--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:25:03.005534Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s namespace: kserve-ci-e2e-test resourceVersion: "22448" uid: 766495ed-b1c2-4ec6-b1f2-17ac15d31a08 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s to ip-10-0-136-15.ec2.internal metadata: creationTimestamp: "2026-07-10T19:25:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:25:03Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s.18c10429566c6cde namespace: kserve-ci-e2e-test resourceVersion: "22452" uid: a5a43dd9-cac7-40e5-b1f5-e555868aa0a8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5495ccf5fb-v5dkr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:25:03Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s namespace: kserve-ci-e2e-test resourceVersion: "22457" uid: 766495ed-b1c2-4ec6-b1f2-17ac15d31a08 kind: Event lastTimestamp: "2026-07-10T19:25:03Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:25:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:25:03Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s.18c1042970454381 namespace: kserve-ci-e2e-test resourceVersion: "22462" uid: f10d2d2b-d579-4d46-86c3-9f4e6657f2d1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:25:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s namespace: kserve-ci-e2e-test resourceVersion: "22449" uid: 766495ed-b1c2-4ec6-b1f2-17ac15d31a08 kind: Event lastTimestamp: "2026-07-10T19:25:03Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:25:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:25:03Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s.18c1042971cf991a namespace: kserve-ci-e2e-test resourceVersion: "22464" uid: 4e48a08d-fc06-467a-95a1-d08c8f28c877 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:25:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s namespace: kserve-ci-e2e-test resourceVersion: "22449" uid: 766495ed-b1c2-4ec6-b1f2-17ac15d31a08 kind: Event lastTimestamp: "2026-07-10T19:25:03Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:25:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:25:03Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s.18c1042974b19138 namespace: kserve-ci-e2e-test resourceVersion: "22466" uid: e8affec8-e129-4b39-9bb6-f160672cd044 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:25:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s namespace: kserve-ci-e2e-test resourceVersion: "22449" uid: 766495ed-b1c2-4ec6-b1f2-17ac15d31a08 kind: Event lastTimestamp: "2026-07-10T19:25:03Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:25:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:25:03Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s.18c104297541aa0f namespace: kserve-ci-e2e-test resourceVersion: "22467" uid: 0455851d-5cca-45f6-9a84-5c6a9dd80fed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:25:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s namespace: kserve-ci-e2e-test resourceVersion: "22449" uid: 766495ed-b1c2-4ec6-b1f2-17ac15d31a08 kind: Event lastTimestamp: "2026-07-10T19:25:14Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-10T19:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:25:14Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s.18c1042bec21dc40 namespace: kserve-ci-e2e-test resourceVersion: "22541" uid: d2a55261-c73f-4ae7-a3c0-e4e487ed1460 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:25:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s namespace: kserve-ci-e2e-test resourceVersion: "22449" uid: 766495ed-b1c2-4ec6-b1f2-17ac15d31a08 kind: Event lastTimestamp: "2026-07-10T19:25:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:25:14Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s.18c1042bef402c04 namespace: kserve-ci-e2e-test resourceVersion: "22544" uid: b8903e6a-9d55-4710-be05-5036d5397beb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:25:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s namespace: kserve-ci-e2e-test resourceVersion: "22449" uid: 766495ed-b1c2-4ec6-b1f2-17ac15d31a08 kind: Event lastTimestamp: "2026-07-10T19:25:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:25:14Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s.18c1042befb5823d namespace: kserve-ci-e2e-test resourceVersion: "22546" uid: c9e12518-6b1f-4d66-8860-85543ec51b8f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:25:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s namespace: kserve-ci-e2e-test resourceVersion: "22449" uid: 766495ed-b1c2-4ec6-b1f2-17ac15d31a08 kind: Event lastTimestamp: "2026-07-10T19:25:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:25:14Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s.18c1042befbf6643 namespace: kserve-ci-e2e-test resourceVersion: "22548" uid: 809b815c-a391-4d69-a0c3-19ac4646f36a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:25:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s namespace: kserve-ci-e2e-test resourceVersion: "22449" uid: 766495ed-b1c2-4ec6-b1f2-17ac15d31a08 kind: Event lastTimestamp: "2026-07-10T19:25:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:25:14Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s.18c1042bf2e1a48d namespace: kserve-ci-e2e-test resourceVersion: "22551" uid: 5e940494-414b-47cd-828d-24adedf9d18b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:25:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s namespace: kserve-ci-e2e-test resourceVersion: "22449" uid: 766495ed-b1c2-4ec6-b1f2-17ac15d31a08 kind: Event lastTimestamp: "2026-07-10T19:25:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:25:14Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s.18c1042bf3585cae namespace: kserve-ci-e2e-test resourceVersion: "22552" uid: 28924b1a-7425-43cb-8e87-c3b40e7ff73d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s namespace: kserve-ci-e2e-test resourceVersion: "22449" uid: 766495ed-b1c2-4ec6-b1f2-17ac15d31a08 kind: Event lastTimestamp: "2026-07-10T19:25:22Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:25:22Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s.18c1042c6410d371 namespace: kserve-ci-e2e-test resourceVersion: "22613" uid: 621e5069-3903-45b0-a598-166c296e7ec9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:25:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s namespace: kserve-ci-e2e-test resourceVersion: "22449" uid: 766495ed-b1c2-4ec6-b1f2-17ac15d31a08 kind: Event lastTimestamp: "2026-07-10T19:25:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:25:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:25:33Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s.18c1043054fa8ca9 namespace: kserve-ci-e2e-test resourceVersion: "22721" uid: 7feea759-6259-4e5e-a2fd-36bd8b54e41e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:25:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s namespace: kserve-ci-e2e-test resourceVersion: "22449" uid: 766495ed-b1c2-4ec6-b1f2-17ac15d31a08 kind: Event lastTimestamp: "2026-07-10T19:25:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:25:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:25:33Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s.18c1043054fb1aa1 namespace: kserve-ci-e2e-test resourceVersion: "22722" uid: 95b03b7f-2c04-42db-90ea-51599cd65816 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:25:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-ts-v1-predictor-89d67dbf7 namespace: kserve-ci-e2e-test resourceVersion: "22437" uid: 23dd0c76-cabd-42a3-a695-aed1db25691c kind: Event lastTimestamp: "2026-07-10T19:25:02Z" message: 'Created pod: isvc-autogluon-ts-v1-predictor-89d67dbf7-8mg7s' metadata: creationTimestamp: "2026-07-10T19:25:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:25:03Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7.18c10429560e2be3 namespace: kserve-ci-e2e-test resourceVersion: "22450" uid: e1619635-e6ad-478b-89d6-5ddcab3b0c16 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:25:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "22435" uid: 52963a8d-7c01-4286-9c83-d9429230321f kind: Event lastTimestamp: "2026-07-10T19:25:02Z" message: Scaled up replica set isvc-autogluon-ts-v1-predictor-89d67dbf7 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:25:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:25:02Z" name: isvc-autogluon-ts-v1-predictor.18c104295113a30c namespace: kserve-ci-e2e-test resourceVersion: "22438" uid: 1021d406-9bf6-455b-bffe-842ba014a083 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:25:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "22436" uid: 22153760-806f-4369-9053-6d04d249f3f3 kind: Event lastTimestamp: "2026-07-10T19:25:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:25:32Z" name: isvc-autogluon-ts-v1-predictor.18c1042ccfe0172f namespace: kserve-ci-e2e-test resourceVersion: "22705" uid: b50d16ea-7413-4bdc-b426-2276670d3a48 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:25:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "22436" uid: 22153760-806f-4369-9053-6d04d249f3f3 kind: Event lastTimestamp: "2026-07-10T19:25:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:25:32Z" name: isvc-autogluon-ts-v1-predictor.18c1042cd03642b1 namespace: kserve-ci-e2e-test resourceVersion: "22707" uid: ca588638-a635-4dac-976a-5f0a137c0c8b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-10T19:25:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "22427" uid: 273ce82c-ab9c-4d85-8cb2-dc5ab204978c kind: Event lastTimestamp: "2026-07-10T19:25:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:25:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:25:32Z" name: isvc-autogluon-ts-v1.18c1042951173b8a namespace: kserve-ci-e2e-test resourceVersion: "22696" uid: 52a898ba-7ecc-4f86-b3da-b5189204930e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:25:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "22427" uid: 273ce82c-ab9c-4d85-8cb2-dc5ab204978c kind: Event lastTimestamp: "2026-07-10T19:25:02Z" message: 'Failed to update status for InferenceService "isvc-autogluon-ts-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:25:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:25:02Z" name: isvc-autogluon-ts-v1.18c1042951bd2da8 namespace: kserve-ci-e2e-test resourceVersion: "22441" uid: d5673b1e-41bc-4005-a68e-81734cfb2565 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:25:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "22427" uid: 273ce82c-ab9c-4d85-8cb2-dc5ab204978c kind: Event lastTimestamp: "2026-07-10T19:25:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:25:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:25:02Z" name: isvc-autogluon-ts-v1.18c1042951bd78bf namespace: kserve-ci-e2e-test resourceVersion: "22442" uid: aaecdeb5-59bf-4b90-a395-d9fb92cdf488 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:25:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "22693" uid: 273ce82c-ab9c-4d85-8cb2-dc5ab204978c kind: Event lastTimestamp: "2026-07-10T19:25:32Z" message: InferenceService [isvc-autogluon-ts-v1] is Ready metadata: creationTimestamp: "2026-07-10T19:25:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:25:32Z" name: isvc-autogluon-ts-v1.18c10430239f4ad0 namespace: kserve-ci-e2e-test resourceVersion: "22694" uid: f1f7cf5f-db8a-45ad-b8f9-4d12a7dd1105 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:15:21.594580Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb namespace: kserve-ci-e2e-test resourceVersion: "18079" uid: 7a12b378-89c2-4525-87b4-91b19d19bd66 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb to ip-10-0-136-15.ec2.internal metadata: creationTimestamp: "2026-07-10T19:15:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:15:21Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb.18c103a1f7a15315 namespace: kserve-ci-e2e-test resourceVersion: "18083" uid: c20ee5d1-7ae0-4fd3-8c2f-0748eaea1858 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5495ccf5fb-v5dkr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:22Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb namespace: kserve-ci-e2e-test resourceVersion: "18089" uid: 7a12b378-89c2-4525-87b4-91b19d19bd66 kind: Event lastTimestamp: "2026-07-10T19:15:22Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:15:22Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb.18c103a21165d1bb namespace: kserve-ci-e2e-test resourceVersion: "18098" uid: 19fb7c71-bc38-408c-9fde-a5c2a9d6d641 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb namespace: kserve-ci-e2e-test resourceVersion: "18080" uid: 7a12b378-89c2-4525-87b4-91b19d19bd66 kind: Event lastTimestamp: "2026-07-10T19:15:22Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:22Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb.18c103a212fa75dd namespace: kserve-ci-e2e-test resourceVersion: "18100" uid: 3436dc0b-cea8-4ad2-9913-6f2f55392d6e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb namespace: kserve-ci-e2e-test resourceVersion: "18080" uid: 7a12b378-89c2-4525-87b4-91b19d19bd66 kind: Event lastTimestamp: "2026-07-10T19:15:22Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:22Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb.18c103a21613fd6d namespace: kserve-ci-e2e-test resourceVersion: "18101" uid: e0d6f976-e1a5-44e3-b894-f43abbeb7855 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb namespace: kserve-ci-e2e-test resourceVersion: "18080" uid: 7a12b378-89c2-4525-87b4-91b19d19bd66 kind: Event lastTimestamp: "2026-07-10T19:15:22Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:15:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:22Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb.18c103a21688cb95 namespace: kserve-ci-e2e-test resourceVersion: "18102" uid: c200610f-55b1-4363-9822-ee79fbfea520 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb namespace: kserve-ci-e2e-test resourceVersion: "18080" uid: 7a12b378-89c2-4525-87b4-91b19d19bd66 kind: Event lastTimestamp: "2026-07-10T19:15:29Z" message: Pulling image "kserve/autogluonserver:latest" metadata: creationTimestamp: "2026-07-10T19:15:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:29Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb.18c103a3d94ff365 namespace: kserve-ci-e2e-test resourceVersion: "18169" uid: 14636aa8-afbc-48f1-b703-7ff1f3297c82 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:17:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb namespace: kserve-ci-e2e-test resourceVersion: "18080" uid: 7a12b378-89c2-4525-87b4-91b19d19bd66 kind: Event lastTimestamp: "2026-07-10T19:17:56Z" message: 'Successfully pulled image "kserve/autogluonserver:latest" in 2m27.152s (2m27.152s including waiting). Image size: 8939191067 bytes.' metadata: creationTimestamp: "2026-07-10T19:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:17:56Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb.18c103c61c50afc9 namespace: kserve-ci-e2e-test resourceVersion: "19191" uid: cf621004-d9a8-4cee-bc32-4176694862eb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:17:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb namespace: kserve-ci-e2e-test resourceVersion: "18080" uid: 7a12b378-89c2-4525-87b4-91b19d19bd66 kind: Event lastTimestamp: "2026-07-10T19:17:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:17:56Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb.18c103c62169ca7f namespace: kserve-ci-e2e-test resourceVersion: "19192" uid: 7e76f292-c5ab-490c-a8b1-83113c423007 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:17:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb namespace: kserve-ci-e2e-test resourceVersion: "18080" uid: 7a12b378-89c2-4525-87b4-91b19d19bd66 kind: Event lastTimestamp: "2026-07-10T19:17:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:17:56Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb.18c103c6228d10a6 namespace: kserve-ci-e2e-test resourceVersion: "19193" uid: f7f6731e-bd2c-4d15-bd6a-4a14d404e08c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:17:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb namespace: kserve-ci-e2e-test resourceVersion: "18080" uid: 7a12b378-89c2-4525-87b4-91b19d19bd66 kind: Event lastTimestamp: "2026-07-10T19:17:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:17:56Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb.18c103c622976b4e namespace: kserve-ci-e2e-test resourceVersion: "19194" uid: ef78403f-21b2-4eca-850e-195632ac2768 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:17:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb namespace: kserve-ci-e2e-test resourceVersion: "18080" uid: 7a12b378-89c2-4525-87b4-91b19d19bd66 kind: Event lastTimestamp: "2026-07-10T19:17:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:17:56Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb.18c103c6266530e7 namespace: kserve-ci-e2e-test resourceVersion: "19195" uid: 822277ad-8f57-4ed6-afbe-0f574f686b94 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:17:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb namespace: kserve-ci-e2e-test resourceVersion: "18080" uid: 7a12b378-89c2-4525-87b4-91b19d19bd66 kind: Event lastTimestamp: "2026-07-10T19:17:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:17:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:17:57Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb.18c103c626dea00e namespace: kserve-ci-e2e-test resourceVersion: "19196" uid: 5cceaa99-655f-4b31-bd49-a052b954575d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:17:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb namespace: kserve-ci-e2e-test resourceVersion: "18080" uid: 7a12b378-89c2-4525-87b4-91b19d19bd66 kind: Event lastTimestamp: "2026-07-10T19:18:04Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:17:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:18:04Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb.18c103c6766d5882 namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: 3a607f1e-c3c5-4dc1-b655-7ee72adc5bf4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:18:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb namespace: kserve-ci-e2e-test resourceVersion: "18080" uid: 7a12b378-89c2-4525-87b4-91b19d19bd66 kind: Event lastTimestamp: "2026-07-10T19:18:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:18:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:18:21Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb.18c103cbe820be8f namespace: kserve-ci-e2e-test resourceVersion: "19395" uid: f205b18a-434e-4409-824f-220c77e2e733 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:18:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb namespace: kserve-ci-e2e-test resourceVersion: "18080" uid: 7a12b378-89c2-4525-87b4-91b19d19bd66 kind: Event lastTimestamp: "2026-07-10T19:18:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:18:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:18:21Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb.18c103cbe821447e namespace: kserve-ci-e2e-test resourceVersion: "19396" uid: dd206f82-6bd3-4727-8f9a-7fb4e8520774 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v1-predictor-7bbbbd79cd namespace: kserve-ci-e2e-test resourceVersion: "18057" uid: 2e511f1d-13d2-40d1-98e8-d05629099ec7 kind: Event lastTimestamp: "2026-07-10T19:15:21Z" message: 'Created pod: isvc-autogluon-v1-predictor-7bbbbd79cd-sqzpb' metadata: creationTimestamp: "2026-07-10T19:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:15:21Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd.18c103a1f73a14f4 namespace: kserve-ci-e2e-test resourceVersion: "18081" uid: 5f9b548c-029f-417e-b0f3-96ccd525ed34 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18051" uid: fa5c8f4d-fa85-44cc-8321-86200d77d76c kind: Event lastTimestamp: "2026-07-10T19:15:21Z" message: Scaled up replica set isvc-autogluon-v1-predictor-7bbbbd79cd from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:15:21Z" name: isvc-autogluon-v1-predictor.18c103a1f2130b27 namespace: kserve-ci-e2e-test resourceVersion: "18059" uid: 564ce5e8-dd4f-44cf-992d-078dfb0be0fb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-10T19:15:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18055" uid: ded9b804-7ca4-4951-bfcc-ba9b6f07bd6d kind: Event lastTimestamp: "2026-07-10T19:18:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:15:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:18:21Z" name: isvc-autogluon-v1-predictor.18c103a5708181df namespace: kserve-ci-e2e-test resourceVersion: "19381" uid: c4c923be-644c-4b3a-85f6-06998018aec3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-10T19:15:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "18055" uid: ded9b804-7ca4-4951-bfcc-ba9b6f07bd6d kind: Event lastTimestamp: "2026-07-10T19:18:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:15:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:18:21Z" name: isvc-autogluon-v1-predictor.18c103a570c38848 namespace: kserve-ci-e2e-test resourceVersion: "19388" uid: 25c8cafb-7199-434c-bedd-d3a79094cb3d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-10T19:15:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "18041" uid: 0c4db050-7ff3-4ab6-9143-db277183a1b9 kind: Event lastTimestamp: "2026-07-10T19:18:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:18:14Z" name: isvc-autogluon-v1.18c103a1f20f637e namespace: kserve-ci-e2e-test resourceVersion: "19330" uid: 5629be52-de2e-41a2-8622-70b2686d7956 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "18041" uid: 0c4db050-7ff3-4ab6-9143-db277183a1b9 kind: Event lastTimestamp: "2026-07-10T19:15:21Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:15:21Z" name: isvc-autogluon-v1.18c103a1f62a4abb namespace: kserve-ci-e2e-test resourceVersion: "18077" uid: a1a6d9b5-5727-4300-94ce-321ccf4fed28 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "18041" uid: 0c4db050-7ff3-4ab6-9143-db277183a1b9 kind: Event lastTimestamp: "2026-07-10T19:15:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:15:21Z" name: isvc-autogluon-v1.18c103a1f62a9ca4 namespace: kserve-ci-e2e-test resourceVersion: "18078" uid: 32c0a03c-9295-475c-b2bb-0237f169c5fa reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:18:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "19327" uid: 0c4db050-7ff3-4ab6-9143-db277183a1b9 kind: Event lastTimestamp: "2026-07-10T19:18:14Z" message: InferenceService [isvc-autogluon-v1] is Ready metadata: creationTimestamp: "2026-07-10T19:18:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:18:14Z" name: isvc-autogluon-v1.18c103ca35f587ee namespace: kserve-ci-e2e-test resourceVersion: "19328" uid: d84d3c0e-5e55-4d41-87ee-1746a1cf945e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:22:47.719478Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5 namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: 008acf47-7844-41e9-a563-528c4836936b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5 to ip-10-0-136-15.ec2.internal metadata: creationTimestamp: "2026-07-10T19:22:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:22:47Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5.18c10409d6bf058d namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: c49c7643-df6a-40fa-a187-e19ebcfe1ee7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5495ccf5fb-v5dkr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:48Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5 namespace: kserve-ci-e2e-test resourceVersion: "21434" uid: 008acf47-7844-41e9-a563-528c4836936b kind: Event lastTimestamp: "2026-07-10T19:22:48Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:22:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:22:48Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5.18c10409f0425f90 namespace: kserve-ci-e2e-test resourceVersion: "21445" uid: 35f2baf8-a006-4883-8c3a-050462659b4f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5 namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 008acf47-7844-41e9-a563-528c4836936b kind: Event lastTimestamp: "2026-07-10T19:22:48Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:22:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:22:48Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5.18c10409f19bfca2 namespace: kserve-ci-e2e-test resourceVersion: "21447" uid: 5dddc4dd-5338-47f5-862b-5852cf7005d5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5 namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 008acf47-7844-41e9-a563-528c4836936b kind: Event lastTimestamp: "2026-07-10T19:22:48Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:22:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:22:48Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5.18c10409f4531ee5 namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: 9f653cf3-9e01-4ce7-a987-9b16adad4e93 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5 namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 008acf47-7844-41e9-a563-528c4836936b kind: Event lastTimestamp: "2026-07-10T19:22:48Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:22:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:22:48Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5.18c10409f4c088e3 namespace: kserve-ci-e2e-test resourceVersion: "21449" uid: f6c61ed3-e775-48a5-855c-313258201f05 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5 namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 008acf47-7844-41e9-a563-528c4836936b kind: Event lastTimestamp: "2026-07-10T19:23:22Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-10T19:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:23:22Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5.18c10411f8ab0dfe namespace: kserve-ci-e2e-test resourceVersion: "21694" uid: c5184235-35ee-408d-a895-e8ca85dbbad0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5 namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 008acf47-7844-41e9-a563-528c4836936b kind: Event lastTimestamp: "2026-07-10T19:23:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:23:22Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5.18c10411fb944d13 namespace: kserve-ci-e2e-test resourceVersion: "21696" uid: 4c70738c-1c3e-4abd-9d49-8fd7c00ce189 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5 namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 008acf47-7844-41e9-a563-528c4836936b kind: Event lastTimestamp: "2026-07-10T19:23:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:23:22Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5.18c10411fc038d7a namespace: kserve-ci-e2e-test resourceVersion: "21698" uid: 4cae98dc-2679-41ce-9eab-b1091d4ed4e9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5 namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 008acf47-7844-41e9-a563-528c4836936b kind: Event lastTimestamp: "2026-07-10T19:23:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:23:22Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5.18c10411fc0e55df namespace: kserve-ci-e2e-test resourceVersion: "21700" uid: 31c503ab-a3d5-4622-b0bd-09a512070d06 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5 namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 008acf47-7844-41e9-a563-528c4836936b kind: Event lastTimestamp: "2026-07-10T19:23:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:23:22Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5.18c10411ff1f12a1 namespace: kserve-ci-e2e-test resourceVersion: "21703" uid: d92b5f80-639f-4f87-b448-4eae84ad4447 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:23:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5 namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 008acf47-7844-41e9-a563-528c4836936b kind: Event lastTimestamp: "2026-07-10T19:23:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:23:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:23:22Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5.18c10411ff8f0797 namespace: kserve-ci-e2e-test resourceVersion: "21704" uid: 949e8e32-e5a2-48bc-b7e9-d2e7adb28592 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:24:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5 namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 008acf47-7844-41e9-a563-528c4836936b kind: Event lastTimestamp: "2026-07-10T19:24:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:24:57Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5.18c104281ff40015 namespace: kserve-ci-e2e-test resourceVersion: "22375" uid: 193a7284-db65-44a7-a676-1b6820efd0c1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:24:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5 namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 008acf47-7844-41e9-a563-528c4836936b kind: Event lastTimestamp: "2026-07-10T19:24:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:24:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:24:57Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5.18c104281ff47a08 namespace: kserve-ci-e2e-test resourceVersion: "22376" uid: 2ec7c440-f0c7-4ade-a526-799a710a9324 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84 namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: fff33903-fbf2-43a0-96c9-4994e5a335c1 kind: Event lastTimestamp: "2026-07-10T19:22:47Z" message: 'Created pod: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-sqnd5' metadata: creationTimestamp: "2026-07-10T19:22:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:47Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84.18c10409d6568bb4 namespace: kserve-ci-e2e-test resourceVersion: "21425" uid: 9d07136c-ab4b-49b6-91f5-c92d65513377 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: e3a08af7-ab69-4d57-9343-4220d6a1390c kind: Event lastTimestamp: "2026-07-10T19:22:47Z" message: Scaled up replica set isvc-autogluon-v2-noslash-predictor-9dcfbbb84 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:22:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:47Z" name: isvc-autogluon-v2-noslash-predictor.18c10409d20b0667 namespace: kserve-ci-e2e-test resourceVersion: "21415" uid: bb0b3f22-e040-4e79-a49c-c9776903f988 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:23:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: 8aeb2e36-97ce-45ac-9215-baf0b2811652 kind: Event lastTimestamp: "2026-07-10T19:24:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:24:02Z" name: isvc-autogluon-v2-noslash-predictor.18c1040d51152a78 namespace: kserve-ci-e2e-test resourceVersion: "21977" uid: e21852a9-04d9-4bf3-ae5b-eac2d86600d6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:23:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: 8aeb2e36-97ce-45ac-9215-baf0b2811652 kind: Event lastTimestamp: "2026-07-10T19:24:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:24:02Z" name: isvc-autogluon-v2-noslash-predictor.18c1040d517fad6b namespace: kserve-ci-e2e-test resourceVersion: "21978" uid: d2a82714-848d-4cff-a7a9-f980786a7dc5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:23:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21553" uid: 8aeb2e36-97ce-45ac-9215-baf0b2811652 kind: Event lastTimestamp: "2026-07-10T19:24:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:23:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:24:47Z" name: isvc-autogluon-v2-noslash-predictor.18c104144f4eeea8 namespace: kserve-ci-e2e-test resourceVersion: "22286" uid: 5e5b5e11-2af7-4f10-ad1c-18b6a6e6cdce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:23:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "21553" uid: 8aeb2e36-97ce-45ac-9215-baf0b2811652 kind: Event lastTimestamp: "2026-07-10T19:24:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:23:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:24:47Z" name: isvc-autogluon-v2-noslash-predictor.18c104144fbcc4ed namespace: kserve-ci-e2e-test resourceVersion: "22287" uid: f1ef325e-9df7-4be5-bf82-5c3db809de0b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-10T19:22:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: 6bcc7036-7eee-487e-8d8f-df0e2a717b7b kind: Event lastTimestamp: "2026-07-10T19:24:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:22:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:24:52Z" name: isvc-autogluon-v2-noslash.18c10409d216d229 namespace: kserve-ci-e2e-test resourceVersion: "22331" uid: 03f29382-6869-4197-aeef-2d50edaab404 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: 6bcc7036-7eee-487e-8d8f-df0e2a717b7b kind: Event lastTimestamp: "2026-07-10T19:22:47Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-noslash": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:22:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:22:47Z" name: isvc-autogluon-v2-noslash.18c10409d2982f7d namespace: kserve-ci-e2e-test resourceVersion: "21419" uid: 54f1ef02-9b9e-453a-a498-c9f848fe48c2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: 6bcc7036-7eee-487e-8d8f-df0e2a717b7b kind: Event lastTimestamp: "2026-07-10T19:22:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:22:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:22:47Z" name: isvc-autogluon-v2-noslash.18c10409d2986e21 namespace: kserve-ci-e2e-test resourceVersion: "21420" uid: 020b7320-14e1-4f44-aae1-b992a86fb919 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:24:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "22328" uid: 6bcc7036-7eee-487e-8d8f-df0e2a717b7b kind: Event lastTimestamp: "2026-07-10T19:24:52Z" message: InferenceService [isvc-autogluon-v2-noslash] is Ready metadata: creationTimestamp: "2026-07-10T19:24:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:24:52Z" name: isvc-autogluon-v2-noslash.18c10426e5112160 namespace: kserve-ci-e2e-test resourceVersion: "22330" uid: 4827c958-2447-41b6-8015-2469266ccfb8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:18:27.013924Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xpnn namespace: kserve-ci-e2e-test resourceVersion: "19455" uid: 8aa0fc2e-9f00-4894-a334-f589f718a7dd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-predictor-67b849d456-5xpnn to ip-10-0-136-15.ec2.internal metadata: creationTimestamp: "2026-07-10T19:18:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:18:27Z" name: isvc-autogluon-v2-predictor-67b849d456-5xpnn.18c103cd237c014e namespace: kserve-ci-e2e-test resourceVersion: "19460" uid: 224c557b-d06d-4352-86cc-e0a7fc052e4d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5495ccf5fb-v5dkr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:18:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xpnn namespace: kserve-ci-e2e-test resourceVersion: "19457" uid: 8aa0fc2e-9f00-4894-a334-f589f718a7dd kind: Event lastTimestamp: "2026-07-10T19:18:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-autogluon-v2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:18:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:18:27Z" name: isvc-autogluon-v2-predictor-67b849d456-5xpnn.18c103cd325a63bb namespace: kserve-ci-e2e-test resourceVersion: "19470" uid: 20e53570-aee6-4813-9ca7-cbe89fc0eeb9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:18:28Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xpnn namespace: kserve-ci-e2e-test resourceVersion: "19464" uid: 8aa0fc2e-9f00-4894-a334-f589f718a7dd kind: Event lastTimestamp: "2026-07-10T19:18:28Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:18:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:18:28Z" name: isvc-autogluon-v2-predictor-67b849d456-5xpnn.18c103cd63416258 namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: 68161e58-e02e-4126-8146-924e124fd630 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:18:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xpnn namespace: kserve-ci-e2e-test resourceVersion: "19457" uid: 8aa0fc2e-9f00-4894-a334-f589f718a7dd kind: Event lastTimestamp: "2026-07-10T19:18:28Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:18:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:18:28Z" name: isvc-autogluon-v2-predictor-67b849d456-5xpnn.18c103cd6496fa7d namespace: kserve-ci-e2e-test resourceVersion: "19480" uid: f559e01d-882e-4edf-b94b-16de3cf93d40 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:18:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xpnn namespace: kserve-ci-e2e-test resourceVersion: "19457" uid: 8aa0fc2e-9f00-4894-a334-f589f718a7dd kind: Event lastTimestamp: "2026-07-10T19:18:28Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:18:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:18:28Z" name: isvc-autogluon-v2-predictor-67b849d456-5xpnn.18c103cd6b14879d namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: 4a555a3e-d0ee-4cf7-8c19-f02b0389b6b9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:18:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xpnn namespace: kserve-ci-e2e-test resourceVersion: "19457" uid: 8aa0fc2e-9f00-4894-a334-f589f718a7dd kind: Event lastTimestamp: "2026-07-10T19:18:28Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:18:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:18:28Z" name: isvc-autogluon-v2-predictor-67b849d456-5xpnn.18c103cd6c648148 namespace: kserve-ci-e2e-test resourceVersion: "19482" uid: 27d9771f-e4f4-4eaa-a126-2844b9192098 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xpnn namespace: kserve-ci-e2e-test resourceVersion: "19457" uid: 8aa0fc2e-9f00-4894-a334-f589f718a7dd kind: Event lastTimestamp: "2026-07-10T19:19:02Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-10T19:19:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:02Z" name: isvc-autogluon-v2-predictor-67b849d456-5xpnn.18c103d56c939926 namespace: kserve-ci-e2e-test resourceVersion: "19730" uid: 690e7b21-66c5-4631-adef-1fd600b8ebc4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xpnn namespace: kserve-ci-e2e-test resourceVersion: "19457" uid: 8aa0fc2e-9f00-4894-a334-f589f718a7dd kind: Event lastTimestamp: "2026-07-10T19:19:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:19:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:02Z" name: isvc-autogluon-v2-predictor-67b849d456-5xpnn.18c103d57099c77b namespace: kserve-ci-e2e-test resourceVersion: "19736" uid: 8c849b93-223b-4fe6-945f-2bccf707518c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xpnn namespace: kserve-ci-e2e-test resourceVersion: "19457" uid: 8aa0fc2e-9f00-4894-a334-f589f718a7dd kind: Event lastTimestamp: "2026-07-10T19:19:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:19:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:02Z" name: isvc-autogluon-v2-predictor-67b849d456-5xpnn.18c103d570fcea70 namespace: kserve-ci-e2e-test resourceVersion: "19737" uid: 35dc8f57-53ef-4621-b2bc-ad812d08d9be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xpnn namespace: kserve-ci-e2e-test resourceVersion: "19457" uid: 8aa0fc2e-9f00-4894-a334-f589f718a7dd kind: Event lastTimestamp: "2026-07-10T19:19:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:19:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:02Z" name: isvc-autogluon-v2-predictor-67b849d456-5xpnn.18c103d57106cf66 namespace: kserve-ci-e2e-test resourceVersion: "19738" uid: 48ac35bd-4fd3-4c17-9d86-13669a81c625 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xpnn namespace: kserve-ci-e2e-test resourceVersion: "19457" uid: 8aa0fc2e-9f00-4894-a334-f589f718a7dd kind: Event lastTimestamp: "2026-07-10T19:19:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:19:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:02Z" name: isvc-autogluon-v2-predictor-67b849d456-5xpnn.18c103d573cdd833 namespace: kserve-ci-e2e-test resourceVersion: "19741" uid: 1a9cf45b-bb61-40a0-97b3-bffbe4bd7e40 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:19:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xpnn namespace: kserve-ci-e2e-test resourceVersion: "19457" uid: 8aa0fc2e-9f00-4894-a334-f589f718a7dd kind: Event lastTimestamp: "2026-07-10T19:19:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:19:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:19:02Z" name: isvc-autogluon-v2-predictor-67b849d456-5xpnn.18c103d5742a736d namespace: kserve-ci-e2e-test resourceVersion: "19742" uid: 1d38f436-2be7-4b85-b375-df80f58f2ae0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xpnn namespace: kserve-ci-e2e-test resourceVersion: "19457" uid: 8aa0fc2e-9f00-4894-a334-f589f718a7dd kind: Event lastTimestamp: "2026-07-10T19:20:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:20:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:47Z" name: isvc-autogluon-v2-predictor-67b849d456-5xpnn.18c103edc9f2524a namespace: kserve-ci-e2e-test resourceVersion: "20484" uid: 57d5a145-792d-4c41-be7b-19568d2ac73a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-5xpnn namespace: kserve-ci-e2e-test resourceVersion: "19457" uid: 8aa0fc2e-9f00-4894-a334-f589f718a7dd kind: Event lastTimestamp: "2026-07-10T19:20:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:20:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:47Z" name: isvc-autogluon-v2-predictor-67b849d456-5xpnn.18c103edc9f3016f namespace: kserve-ci-e2e-test resourceVersion: "20485" uid: e4be61ba-c527-45c6-ba75-933467523e3e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:18:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-predictor-67b849d456 namespace: kserve-ci-e2e-test resourceVersion: "19446" uid: 3b8568d8-ab4d-44ad-8282-c645cb88675a kind: Event lastTimestamp: "2026-07-10T19:18:27Z" message: 'Created pod: isvc-autogluon-v2-predictor-67b849d456-5xpnn' metadata: creationTimestamp: "2026-07-10T19:18:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:18:27Z" name: isvc-autogluon-v2-predictor-67b849d456.18c103cd231515fb namespace: kserve-ci-e2e-test resourceVersion: "19456" uid: 9228d8db-456b-42a3-a111-b55073d92b35 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:18:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19444" uid: 6cd6055a-099b-471a-ab17-12e270a961e5 kind: Event lastTimestamp: "2026-07-10T19:18:26Z" message: Scaled up replica set isvc-autogluon-v2-predictor-67b849d456 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:18:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:18:26Z" name: isvc-autogluon-v2-predictor.18c103cd1ec937cc namespace: kserve-ci-e2e-test resourceVersion: "19447" uid: a0caf6c8-603e-4e4f-8dd2-3e43be996e88 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:18:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19445" uid: 6951cb35-0336-443e-a68a-c4efa3337713 kind: Event lastTimestamp: "2026-07-10T19:19:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:18:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:19:11Z" name: isvc-autogluon-v2-predictor.18c103d09d5c963f namespace: kserve-ci-e2e-test resourceVersion: "19804" uid: 58f99f08-3e01-4bbd-97f0-3875295fa864 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:18:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19445" uid: 6951cb35-0336-443e-a68a-c4efa3337713 kind: Event lastTimestamp: "2026-07-10T19:19:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:18:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:19:11Z" name: isvc-autogluon-v2-predictor.18c103d09dc2b738 namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: 232bf525-613f-4640-9df2-2231b81723a0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:19:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19587" uid: 6951cb35-0336-443e-a68a-c4efa3337713 kind: Event lastTimestamp: "2026-07-10T19:20:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:42Z" name: isvc-autogluon-v2-predictor.18c103db1aeb5614 namespace: kserve-ci-e2e-test resourceVersion: "20431" uid: 40191398-eb55-4b17-b193-078cdb0cf7ce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-10T19:19:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19587" uid: 6951cb35-0336-443e-a68a-c4efa3337713 kind: Event lastTimestamp: "2026-07-10T19:20:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:42Z" name: isvc-autogluon-v2-predictor.18c103db1b5c2c9d namespace: kserve-ci-e2e-test resourceVersion: "20432" uid: 818fad42-0adc-4efa-a151-263197ac463e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:20:52.354616Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt namespace: kserve-ci-e2e-test resourceVersion: "20543" uid: c1ab6420-329d-4ad7-9529-c68ef65af108 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt to ip-10-0-136-15.ec2.internal metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:20:52Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt.18c103eefa76e5d5 namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: abd9a5e9-fb4d-4741-b3be-6b3f0f682bb9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5495ccf5fb-v5dkr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: c1ab6420-329d-4ad7-9529-c68ef65af108 kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:20:52Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt.18c103ef13a0cbd9 namespace: kserve-ci-e2e-test resourceVersion: "20560" uid: e0ae56d1-ff94-4079-b495-9af632f3d362 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: c1ab6420-329d-4ad7-9529-c68ef65af108 kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:52Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt.18c103ef1525d836 namespace: kserve-ci-e2e-test resourceVersion: "20562" uid: 04a53b3c-bd47-49ae-8ed1-49b45f2ece2c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: c1ab6420-329d-4ad7-9529-c68ef65af108 kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:52Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt.18c103ef1849d7da namespace: kserve-ci-e2e-test resourceVersion: "20563" uid: c6755c8f-5804-42e6-b1ef-66a05db890fc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: c1ab6420-329d-4ad7-9529-c68ef65af108 kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:52Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt.18c103ef18acb628 namespace: kserve-ci-e2e-test resourceVersion: "20564" uid: f899c977-cd29-492a-aeec-4e8ac2f1281b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: c1ab6420-329d-4ad7-9529-c68ef65af108 kind: Event lastTimestamp: "2026-07-10T19:20:58Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-10T19:20:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:58Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt.18c103f04efd8e9a namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: 5eb8f4fc-22bd-4fc0-8c7b-f62625ca201e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: c1ab6420-329d-4ad7-9529-c68ef65af108 kind: Event lastTimestamp: "2026-07-10T19:20:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:20:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:58Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt.18c103f051dc35a2 namespace: kserve-ci-e2e-test resourceVersion: "20613" uid: 70fd59ea-906f-4e7b-a492-3acc6b3d030b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: c1ab6420-329d-4ad7-9529-c68ef65af108 kind: Event lastTimestamp: "2026-07-10T19:20:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:20:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:58Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt.18c103f0523a5c27 namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: a9e025d8-c38f-4a2a-bdb4-32fe206db240 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: c1ab6420-329d-4ad7-9529-c68ef65af108 kind: Event lastTimestamp: "2026-07-10T19:20:58Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:20:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:58Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt.18c103f05243cc2e namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: a12be3f3-dc73-43d4-9456-d4721cbb2fcc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: c1ab6420-329d-4ad7-9529-c68ef65af108 kind: Event lastTimestamp: "2026-07-10T19:20:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:20:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:58Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt.18c103f05517da31 namespace: kserve-ci-e2e-test resourceVersion: "20619" uid: 7f33b323-100f-4039-8d8b-bbecf3496d4e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: c1ab6420-329d-4ad7-9529-c68ef65af108 kind: Event lastTimestamp: "2026-07-10T19:20:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:20:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:20:58Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt.18c103f05579cec4 namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 3fbb49c7-c708-444c-b036-862d63244255 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: c1ab6420-329d-4ad7-9529-c68ef65af108 kind: Event lastTimestamp: "2026-07-10T19:22:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:22:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:22:42Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt.18c104089eb6edee namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: 69a49edb-dee8-4d7b-a67f-ee7696dd8cd8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: c1ab6420-329d-4ad7-9529-c68ef65af108 kind: Event lastTimestamp: "2026-07-10T19:22:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:22:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:22:42Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt.18c104089eb77731 namespace: kserve-ci-e2e-test resourceVersion: "21366" uid: 552eb20c-16a0-4ae2-b83d-956577e41fd4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-variants-predictor-c6767c94c namespace: kserve-ci-e2e-test resourceVersion: "20533" uid: 8f21a6fa-eb50-4a76-bb4e-e1b5bf104489 kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: 'Created pod: isvc-autogluon-v2-variants-predictor-c6767c94c-nc9vt' metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:52Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c.18c103eefa0d1a33 namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: 9ddc81e1-ebba-43ef-b7e5-5a32652a012e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20531" uid: 996101b3-7e6e-4855-8228-f929efa36ef0 kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: Scaled up replica set isvc-autogluon-v2-variants-predictor-c6767c94c from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:20:52Z" name: isvc-autogluon-v2-variants-predictor.18c103eef5070d59 namespace: kserve-ci-e2e-test resourceVersion: "20534" uid: 09be77c2-df36-4c00-8291-35a37cd0d62e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:21:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 38dbe9c0-c391-4caf-8db8-af1fcbc523ca kind: Event lastTimestamp: "2026-07-10T19:21:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:21:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:22Z" name: isvc-autogluon-v2-variants-predictor.18c103f2738e72e0 namespace: kserve-ci-e2e-test resourceVersion: "20787" uid: f07819e6-f9d2-449f-8390-ad6b59cfe27e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:21:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20532" uid: 38dbe9c0-c391-4caf-8db8-af1fcbc523ca kind: Event lastTimestamp: "2026-07-10T19:21:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:21:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:21:22Z" name: isvc-autogluon-v2-variants-predictor.18c103f273f9d1fe namespace: kserve-ci-e2e-test resourceVersion: "20788" uid: 7695b62e-666b-4fc0-91b8-e728055f4279 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-10T19:21:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 38dbe9c0-c391-4caf-8db8-af1fcbc523ca kind: Event lastTimestamp: "2026-07-10T19:22:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:21:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:37Z" name: isvc-autogluon-v2-variants-predictor.18c103f971b9c70c namespace: kserve-ci-e2e-test resourceVersion: "21312" uid: fcd69bc4-ec92-4df7-afbf-c3254bb90c9d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-10T19:21:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 38dbe9c0-c391-4caf-8db8-af1fcbc523ca kind: Event lastTimestamp: "2026-07-10T19:22:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:21:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:22:37Z" name: isvc-autogluon-v2-variants-predictor.18c103f972240dc5 namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: f02f37e8-bdb0-48cf-8281-990e5111290c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: f6cb4ae6-7968-41ed-b8b4-5429ff589e15 kind: Event lastTimestamp: "2026-07-10T19:22:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:22:36Z" name: isvc-autogluon-v2-variants.18c103eef50c852a namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: dbb1f1a4-b3c2-4d60-8c80-7a2300aae2b6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: f6cb4ae6-7968-41ed-b8b4-5429ff589e15 kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-variants": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:52Z" name: isvc-autogluon-v2-variants.18c103eef596f441 namespace: kserve-ci-e2e-test resourceVersion: "20554" uid: 31016532-1c9b-4131-8c29-623ceeee4d8c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:20:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: f6cb4ae6-7968-41ed-b8b4-5429ff589e15 kind: Event lastTimestamp: "2026-07-10T19:20:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:52Z" name: isvc-autogluon-v2-variants.18c103eef59735e5 namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: 356532f1-93a2-4006-b1b6-2400f563e98d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:22:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "21306" uid: f6cb4ae6-7968-41ed-b8b4-5429ff589e15 kind: Event lastTimestamp: "2026-07-10T19:22:36Z" message: InferenceService [isvc-autogluon-v2-variants] is Ready metadata: creationTimestamp: "2026-07-10T19:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:22:36Z" name: isvc-autogluon-v2-variants.18c104073aa7242a namespace: kserve-ci-e2e-test resourceVersion: "21307" uid: 4e48ba52-7d5f-4c92-a99e-e318e0c28dea reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-10T19:18:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "19436" uid: 26961d37-c5bf-4742-9908-2815d116a5ec kind: Event lastTimestamp: "2026-07-10T19:20:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:18:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:41Z" name: isvc-autogluon-v2.18c103cd1ecc5b26 namespace: kserve-ci-e2e-test resourceVersion: "20427" uid: 0aa125d3-f477-43ad-803c-c7c3027b63dc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:18:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "19436" uid: 26961d37-c5bf-4742-9908-2815d116a5ec kind: Event lastTimestamp: "2026-07-10T19:18:26Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:18:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:18:26Z" name: isvc-autogluon-v2.18c103cd1f605b92 namespace: kserve-ci-e2e-test resourceVersion: "19450" uid: dc157506-ccd5-4ac0-a48c-f049138925e1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:18:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "19436" uid: 26961d37-c5bf-4742-9908-2815d116a5ec kind: Event lastTimestamp: "2026-07-10T19:18:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:18:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:18:26Z" name: isvc-autogluon-v2.18c103cd1f609d74 namespace: kserve-ci-e2e-test resourceVersion: "19451" uid: 047806e0-d20a-4395-ba12-22c703009386 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:20:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "20424" uid: 26961d37-c5bf-4742-9908-2815d116a5ec kind: Event lastTimestamp: "2026-07-10T19:20:41Z" message: InferenceService [isvc-autogluon-v2] is Ready metadata: creationTimestamp: "2026-07-10T19:20:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:20:41Z" name: isvc-autogluon-v2.18c103ec747ddec4 namespace: kserve-ci-e2e-test resourceVersion: "20426" uid: 75fdebb2-5aeb-4842-a12c-a3c9ba820c5d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:13:46.228402Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-predictor-5bffdc48ff-f9khp to ip-10-0-136-15.ec2.internal metadata: creationTimestamp: "2026-07-10T19:13:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:13:46Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c1038bc35cef90 namespace: kserve-ci-e2e-test resourceVersion: "17288" uid: 80d8d873-351b-460f-87d4-89ebf5be0e72 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5495ccf5fb-v5dkr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:46Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: "2026-07-10T19:13:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:46Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c1038bced2b919 namespace: kserve-ci-e2e-test resourceVersion: "17300" uid: b1069b7f-9f2d-4f6d-8438-5efcecb9ebff reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:47Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17293" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: "2026-07-10T19:13:47Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:13:47Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c1038c008c4b24 namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: 13d338b9-73cb-47f8-bee7-345ece3f5313 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: "2026-07-10T19:13:47Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:47Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c1038c028983a9 namespace: kserve-ci-e2e-test resourceVersion: "17310" uid: ac162809-e46c-4cc9-bb05-81ce9f1fcff3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: "2026-07-10T19:13:47Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:47Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c1038c05f1e81b namespace: kserve-ci-e2e-test resourceVersion: "17311" uid: 596dd14b-8f55-4b2a-9f9e-11adefc0fbff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: "2026-07-10T19:13:47Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:13:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:47Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c1038c0661f977 namespace: kserve-ci-e2e-test resourceVersion: "17312" uid: 2ea46693-2dfb-4934-8c60-d777868db460 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: "2026-07-10T19:13:51Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:51Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c1038cee8f3750 namespace: kserve-ci-e2e-test resourceVersion: "17346" uid: 7af08f3d-11e5-4bc9-978f-255f24cccece reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: "2026-07-10T19:13:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:51Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c1038cf1cbc182 namespace: kserve-ci-e2e-test resourceVersion: "17348" uid: a3b547ec-f187-488f-8121-94d172ae5dec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: "2026-07-10T19:13:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:51Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c1038cf2409552 namespace: kserve-ci-e2e-test resourceVersion: "17351" uid: 73e0f8a6-f916-43d9-91f7-6d5fe68db544 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: "2026-07-10T19:13:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:51Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c1038cf2498e5e namespace: kserve-ci-e2e-test resourceVersion: "17352" uid: 2d10eb7e-a553-4ccc-a035-8b8f94a7d475 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: "2026-07-10T19:13:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:51Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c1038cf53c1fe8 namespace: kserve-ci-e2e-test resourceVersion: "17354" uid: d9c853d0-bc3d-44f3-9af6-b4c0a73db617 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: "2026-07-10T19:13:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:51Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c1038cf5ac5c65 namespace: kserve-ci-e2e-test resourceVersion: "17355" uid: a1dad939-7810-4df8-aaa8-35a46d18711f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: "2026-07-10T19:13:51Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:4c6a130c213136d803ee987407e3094d900670800a73507fd61c7fd7f0573052" already present on machine metadata: creationTimestamp: "2026-07-10T19:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:51Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c1038cf5b5e626 namespace: kserve-ci-e2e-test resourceVersion: "17356" uid: 65a54dc1-750b-4e32-a0c8-546ad689d632 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: "2026-07-10T19:13:51Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-10T19:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:51Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c1038cf88bfaf5 namespace: kserve-ci-e2e-test resourceVersion: "17357" uid: 5c6afa93-666b-413d-8e28-8b89344a2d01 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: "2026-07-10T19:13:51Z" message: Started container agent metadata: creationTimestamp: "2026-07-10T19:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:51Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c1038cf910ea65 namespace: kserve-ci-e2e-test resourceVersion: "17358" uid: f40267a6-2c29-4a5c-861f-d41c998e35e0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-10T19:13:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: "2026-07-10T19:15:28Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:28Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c1038d2a9b0d62 namespace: kserve-ci-e2e-test resourceVersion: "18155" uid: ea44f2b7-b78b-4995-bfcb-d53a547fbe83 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-10T19:13:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: "2026-07-10T19:15:28Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T19:13:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:28Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c1038d2aa53ce3 namespace: kserve-ci-e2e-test resourceVersion: "18156" uid: 18595afc-a32e-41ba-bcb1-494311ec4f97 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: "2026-07-10T19:15:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:21Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c103a1ee12aaa6 namespace: kserve-ci-e2e-test resourceVersion: "18040" uid: cbd5affa-28fd-41b2-b713-d006ff87468f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: "2026-07-10T19:15:21Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-10T19:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:21Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c103a1ee12f59d namespace: kserve-ci-e2e-test resourceVersion: "18042" uid: 90cfb9ea-d43d-49b4-aca4-db6f97c85d5e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: "2026-07-10T19:15:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:21Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c103a1ee131d35 namespace: kserve-ci-e2e-test resourceVersion: "18043" uid: 2590b785-7220-48b4-a77c-16c25a6328df reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:15:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-5bffdc48ff-f9khp namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 5a130c7f-0bd5-44ba-a93d-bd395b7521eb kind: Event lastTimestamp: "2026-07-10T19:15:38Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:15:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:38Z" name: isvc-logger-predictor-5bffdc48ff-f9khp.18c103a25ade2843 namespace: kserve-ci-e2e-test resourceVersion: "18238" uid: fb154242-6d45-4e8b-9571-552ccaa2bbc1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-predictor-5bffdc48ff namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2a1c8e14-eaae-4b82-850a-f2577a627e51 kind: Event lastTimestamp: "2026-07-10T19:13:46Z" message: 'Created pod: isvc-logger-predictor-5bffdc48ff-f9khp' metadata: creationTimestamp: "2026-07-10T19:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:13:46Z" name: isvc-logger-predictor-5bffdc48ff.18c1038bc2f3925c namespace: kserve-ci-e2e-test resourceVersion: "17284" uid: aba40e43-6c2e-4697-89be-152c61478e21 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "17270" uid: 8fb13bb6-231f-45b0-bb6c-7b676b62f132 kind: Event lastTimestamp: "2026-07-10T19:13:46Z" message: Scaled up replica set isvc-logger-predictor-5bffdc48ff from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:13:46Z" name: isvc-logger-predictor.18c1038bbfbf5726 namespace: kserve-ci-e2e-test resourceVersion: "17273" uid: 94f0db9f-bdad-4fa8-818f-8c4d7d87fbb0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:14:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "17271" uid: ce4ec7f3-efb1-40d6-bb29-f58769f76744 kind: Event lastTimestamp: "2026-07-10T19:14:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:14:31Z" name: isvc-logger-predictor.18c1038f3e47f4b2 namespace: kserve-ci-e2e-test resourceVersion: "17652" uid: eeb65c01-2032-4f75-9fc3-ff1bc66d929b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:14:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "17271" uid: ce4ec7f3-efb1-40d6-bb29-f58769f76744 kind: Event lastTimestamp: "2026-07-10T19:14:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:14:31Z" name: isvc-logger-predictor.18c1038f3eab0f31 namespace: kserve-ci-e2e-test resourceVersion: "17653" uid: 196d558a-3cd3-48a0-b723-9f96160ce959 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:14:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "17437" uid: ce4ec7f3-efb1-40d6-bb29-f58769f76744 kind: Event lastTimestamp: "2026-07-10T19:15:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:15:16Z" name: isvc-logger-predictor.18c10399bb81f8c8 namespace: kserve-ci-e2e-test resourceVersion: "17981" uid: 84ab4550-fed3-4cdb-931c-70000e2c0fc2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:14:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "17437" uid: ce4ec7f3-efb1-40d6-bb29-f58769f76744 kind: Event lastTimestamp: "2026-07-10T19:15:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:15:16Z" name: isvc-logger-predictor.18c10399bbeb1771 namespace: kserve-ci-e2e-test resourceVersion: "17982" uid: e67e4790-a7ed-4bb1-80e4-af739d5257da reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-10T19:13:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "17263" uid: 32509216-97fb-4491-9768-f88c64d7b659 kind: Event lastTimestamp: "2026-07-10T19:15:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:15:08Z" name: isvc-logger.18c1038bbfc4ec33 namespace: kserve-ci-e2e-test resourceVersion: "17929" uid: 0e8543a4-9bb3-456c-b6f4-2cae88ce25eb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "17263" uid: 32509216-97fb-4491-9768-f88c64d7b659 kind: Event lastTimestamp: "2026-07-10T19:13:46Z" message: 'Failed to update status for InferenceService "isvc-logger": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:13:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:13:46Z" name: isvc-logger.18c1038bc03e893f namespace: kserve-ci-e2e-test resourceVersion: "17277" uid: 7ec008c8-627f-422c-8449-df5e5723700f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "17926" uid: 32509216-97fb-4491-9768-f88c64d7b659 kind: Event lastTimestamp: "2026-07-10T19:15:08Z" message: InferenceService [isvc-logger] is Ready metadata: creationTimestamp: "2026-07-10T19:15:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:15:08Z" name: isvc-logger.18c1039ee430a251 namespace: kserve-ci-e2e-test resourceVersion: "17928" uid: ff7161a3-b168-4d9b-b224-0bc69433cbee reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:11:50.998209Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16297" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm to ip-10-0-136-15.ec2.internal metadata: creationTimestamp: "2026-07-10T19:11:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:11:50Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c10370ef1bb1d8 namespace: kserve-ci-e2e-test resourceVersion: "16301" uid: bb233193-e938-4197-ba22-a4a5afda8c5c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5495ccf5fb-v5dkr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: "2026-07-10T19:11:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-batcher-custom-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:51Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c10370f66e1c83 namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: fca15b0b-a80e-408c-a9dc-71e1c110a7ec reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16308" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: "2026-07-10T19:11:52Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:11:52Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c103712c1b65ce namespace: kserve-ci-e2e-test resourceVersion: "16324" uid: aebc8a3e-ed4d-4d42-9520-3bccb40b8b31 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: "2026-07-10T19:11:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:52Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c103712e9f647d namespace: kserve-ci-e2e-test resourceVersion: "16326" uid: bf323507-9cc0-44cd-9ca8-da772bbd31e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: "2026-07-10T19:11:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:52Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c1037131556e7f namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: 1d4253ec-54e0-4911-b118-648eb9ae74a2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: "2026-07-10T19:11:52Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:52Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c1037131bc2c2d namespace: kserve-ci-e2e-test resourceVersion: "16328" uid: ddffd549-b2f6-40b5-9e46-4edfd88e2ed5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: "2026-07-10T19:11:56Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1722" already present on machine metadata: creationTimestamp: "2026-07-10T19:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:56Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c1037246095733 namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: 9a0f3494-7746-4f3e-a8d6-85256d6f67ee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: "2026-07-10T19:11:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:56Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c103724933554a namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: 5b043a83-28fc-4281-967c-de71c9741168 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: "2026-07-10T19:11:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:56Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c103724994bd82 namespace: kserve-ci-e2e-test resourceVersion: "16370" uid: 576315eb-b402-4eea-ad21-1c59e7e18ecd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: "2026-07-10T19:11:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:56Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c10372499e7da6 namespace: kserve-ci-e2e-test resourceVersion: "16373" uid: 11a49ef8-a074-460a-a114-3e360a6d2c97 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: "2026-07-10T19:11:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:56Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c103724c5b51a8 namespace: kserve-ci-e2e-test resourceVersion: "16374" uid: afdffea4-a1b2-4c96-ac67-505bbfae30d9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: "2026-07-10T19:11:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:56Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c103724cc00cab namespace: kserve-ci-e2e-test resourceVersion: "16376" uid: 60d7c7df-ef6d-4174-9f73-ef0f4d8f6182 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: "2026-07-10T19:11:56Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:4c6a130c213136d803ee987407e3094d900670800a73507fd61c7fd7f0573052" already present on machine metadata: creationTimestamp: "2026-07-10T19:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:56Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c103724cc9ffdf namespace: kserve-ci-e2e-test resourceVersion: "16377" uid: aba8d9c7-6926-453d-90c4-b0e366dad0a3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: "2026-07-10T19:11:56Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-10T19:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:56Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c103724fbacaed namespace: kserve-ci-e2e-test resourceVersion: "16378" uid: cc142bfa-5c24-4d6d-baf4-6dab0d7e78e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: "2026-07-10T19:11:56Z" message: Started container agent metadata: creationTimestamp: "2026-07-10T19:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:56Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c1037250208f66 namespace: kserve-ci-e2e-test resourceVersion: "16379" uid: edade1ba-b8db-4c35-b9eb-71eea32f5c57 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-10T19:11:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: "2026-07-10T19:13:44Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:5000: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:11:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:44Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c10372bde87991 namespace: kserve-ci-e2e-test resourceVersion: "17247" uid: 63d20ae2-609c-4867-bcb5-5e2acc1295c7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-10T19:11:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: "2026-07-10T19:13:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T19:11:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:34Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c10372bdf2ca05 namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: 7c843bd3-f042-4c01-8da9-55697991ed7d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: "2026-07-10T19:13:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:26Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c10387137052c5 namespace: kserve-ci-e2e-test resourceVersion: "17064" uid: 11657654-cfaf-4339-9f4f-9163daaf67d2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: "2026-07-10T19:13:26Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-10T19:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:26Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c103871371abac namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: 694a4d78-6ad5-412d-bb89-d21045c105cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: "2026-07-10T19:13:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:26Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c103871371c04f namespace: kserve-ci-e2e-test resourceVersion: "17067" uid: 6dbeb8eb-94e3-444b-8500-0d691eed0907 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:13:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 61245d75-9889-4b3f-ba6c-4b671b6607c9 kind: Event lastTimestamp: "2026-07-10T19:13:39Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp 10.133.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:13:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:39Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm.18c10387ee1ecb78 namespace: kserve-ci-e2e-test resourceVersion: "17208" uid: 410b65f4-8ced-47af-8681-160fb66ec0cd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-custom-predictor-578cb45d67 namespace: kserve-ci-e2e-test resourceVersion: "16285" uid: c637d58e-1163-4289-b451-daa5458d851d kind: Event lastTimestamp: "2026-07-10T19:11:50Z" message: 'Created pod: isvc-sklearn-batcher-custom-predictor-578cb45d67-d85mm' metadata: creationTimestamp: "2026-07-10T19:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:50Z" name: isvc-sklearn-batcher-custom-predictor-578cb45d67.18c10370ee9f0e99 namespace: kserve-ci-e2e-test resourceVersion: "16298" uid: 978c1495-24fe-404f-8c84-4361c6c012fb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "16281" uid: ce7a966e-35f0-4092-9ba0-2260d3e38a53 kind: Event lastTimestamp: "2026-07-10T19:11:50Z" message: Scaled up replica set isvc-sklearn-batcher-custom-predictor-578cb45d67 from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:50Z" name: isvc-sklearn-batcher-custom-predictor.18c10370e991f144 namespace: kserve-ci-e2e-test resourceVersion: "16287" uid: 3d654615-f0fa-4786-b4ed-0e68c5ec3d6d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:12:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: 960c1048-278a-4f48-addb-5bb9af56b670 kind: Event lastTimestamp: "2026-07-10T19:12:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:12:35Z" name: isvc-sklearn-batcher-custom-predictor.18c103746834a0c9 namespace: kserve-ci-e2e-test resourceVersion: "16676" uid: ef5553b5-8c85-4505-9eb8-e99dc89e6f8c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:12:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: 960c1048-278a-4f48-addb-5bb9af56b670 kind: Event lastTimestamp: "2026-07-10T19:12:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:12:35Z" name: isvc-sklearn-batcher-custom-predictor.18c103746895720a namespace: kserve-ci-e2e-test resourceVersion: "16677" uid: e7199800-5da9-40d7-bd73-2a540b17df4d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:12:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "16460" uid: 960c1048-278a-4f48-addb-5bb9af56b670 kind: Event lastTimestamp: "2026-07-10T19:13:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:12:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:13:21Z" name: isvc-sklearn-batcher-custom-predictor.18c1037ee5e1c614 namespace: kserve-ci-e2e-test resourceVersion: "17002" uid: fd74508b-3e36-4d25-9669-9c9d4cfffa72 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:12:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "16460" uid: 960c1048-278a-4f48-addb-5bb9af56b670 kind: Event lastTimestamp: "2026-07-10T19:13:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:12:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:13:21Z" name: isvc-sklearn-batcher-custom-predictor.18c1037ee643c631 namespace: kserve-ci-e2e-test resourceVersion: "17003" uid: 276cd25c-5085-4c1f-b14d-b19885ad75ef reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-10T19:11:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "16257" uid: c68d73b3-15ad-4027-9a35-b1df79705c39 kind: Event lastTimestamp: "2026-07-10T19:13:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:13:14Z" name: isvc-sklearn-batcher-custom.18c10370e9976b1a namespace: kserve-ci-e2e-test resourceVersion: "16951" uid: ba05429a-1962-408d-8d4f-62e64190b137 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:11:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "16257" uid: c68d73b3-15ad-4027-9a35-b1df79705c39 kind: Event lastTimestamp: "2026-07-10T19:11:51Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher-custom": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher-custom": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:51Z" name: isvc-sklearn-batcher-custom.18c10370e9f4611b namespace: kserve-ci-e2e-test resourceVersion: "16306" uid: 59c07567-1885-445e-8d03-26d7bcffb128 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "16265" uid: c68d73b3-15ad-4027-9a35-b1df79705c39 kind: Event lastTimestamp: "2026-07-10T19:11:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher-custom": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:51Z" name: isvc-sklearn-batcher-custom.18c10370f01ae394 namespace: kserve-ci-e2e-test resourceVersion: "16307" uid: d0604eea-036e-4822-add4-fd169d5ab562 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "16949" uid: c68d73b3-15ad-4027-9a35-b1df79705c39 kind: Event lastTimestamp: "2026-07-10T19:13:14Z" message: InferenceService [isvc-sklearn-batcher-custom] is Ready metadata: creationTimestamp: "2026-07-10T19:13:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:13:14Z" name: isvc-sklearn-batcher-custom.18c1038461b418f0 namespace: kserve-ci-e2e-test resourceVersion: "16950" uid: 9ec23279-3e37-4258-a627-24e58f6a06c8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:10:05.780331Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15451" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l to ip-10-0-136-15.ec2.internal metadata: creationTimestamp: "2026-07-10T19:10:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:10:05Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c103586fa249e9 namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: f758eb13-2eb5-4dbd-a24f-0b8b46d0bb60 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5495ccf5fb-v5dkr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15464" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:10:06Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:10:06Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c10358893188a2 namespace: kserve-ci-e2e-test resourceVersion: "15472" uid: ef9e2fd1-0786-4fa7-875e-1c49eb0b2b77 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:10:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7893a590791cc9f33e3a3207dcc1d0843778bbf1aee8b6f22a63e3aa3b91a2e1" already present on machine metadata: creationTimestamp: "2026-07-10T19:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:06Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c103588abfb867 namespace: kserve-ci-e2e-test resourceVersion: "15474" uid: 152c2380-d0af-4354-847c-aeee99839f27 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:10:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-10T19:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:06Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c103588da3d458 namespace: kserve-ci-e2e-test resourceVersion: "15475" uid: 53ac8820-4acf-48f2-90b7-65ce60f41614 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:10:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-10T19:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:06Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c103588e12080f namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: 306b06bc-c546-49cb-af67-de4f547cd96b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:10:10Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1722" metadata: creationTimestamp: "2026-07-10T19:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:10Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c103597d4f7db8 namespace: kserve-ci-e2e-test resourceVersion: "15510" uid: 665019c8-3fd2-4689-a135-5796f256cd29 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:10:25Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1722" in 14.711s (14.711s including waiting). Image size: 1555838884 bytes.' metadata: creationTimestamp: "2026-07-10T19:10:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:25Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c1035cea310a2f namespace: kserve-ci-e2e-test resourceVersion: "15621" uid: a4485d77-f0c8-484b-b0db-df7e730ae8b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:10:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:10:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:25Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c1035cedc597b0 namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: 85161632-c55d-4e72-bf28-f02d6638a836 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:10:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:10:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:25Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c1035cee357e10 namespace: kserve-ci-e2e-test resourceVersion: "15623" uid: f33da507-e665-48a5-a889-58e6aaaa71a5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:10:25Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-10T19:10:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:25Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c1035cee410df2 namespace: kserve-ci-e2e-test resourceVersion: "15624" uid: a94bd156-740f-4dbb-841a-7b08d55f20df reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:10:27Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.339s (2.339s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-10T19:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:27Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c1035d79b5c1eb namespace: kserve-ci-e2e-test resourceVersion: "15639" uid: 3c837e84-5c6a-4bbb-9886-cdb9d01fbb48 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:10:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:27Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c1035d7d254771 namespace: kserve-ci-e2e-test resourceVersion: "15640" uid: d67d4542-561b-49c1-b1e2-1d790dfee1fd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:10:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:27Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c1035d7d981276 namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: 0edccc3d-949f-49be-97e4-00b20763d71e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:10:27Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:4c6a130c213136d803ee987407e3094d900670800a73507fd61c7fd7f0573052" metadata: creationTimestamp: "2026-07-10T19:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:27Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c1035d7da4dd0a namespace: kserve-ci-e2e-test resourceVersion: "15643" uid: 88898fa9-8e93-45bb-ac7e-a1694df4e157 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:10:29Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:4c6a130c213136d803ee987407e3094d900670800a73507fd61c7fd7f0573052" in 1.948s (1.948s including waiting). Image size: 239997016 bytes.' metadata: creationTimestamp: "2026-07-10T19:10:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:29Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c1035df1cecdde namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: c087563a-80dd-47b3-b0e2-7879ca1235cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:10:29Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-10T19:10:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:29Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c1035df503249b namespace: kserve-ci-e2e-test resourceVersion: "15658" uid: d54f2a59-1a82-465d-9d8d-86189ab82ef6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:10:29Z" message: Started container agent metadata: creationTimestamp: "2026-07-10T19:10:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:10:29Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c1035df571a467 namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: b1f50abe-d992-4128-87fb-573229c0cbab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-10T19:10:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:11:53Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:10:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:53Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c1035e2bbb9b63 namespace: kserve-ci-e2e-test resourceVersion: "16345" uid: b2b11e8c-2b12-4bd6-aa0a-312785c5a9ac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-10T19:10:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:12:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-10T19:10:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:12:03Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c1035e679794ad namespace: kserve-ci-e2e-test resourceVersion: "16435" uid: b316120b-2626-4096-8ea6-879ad700b09e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:11:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:50Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c10370e82d7307 namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: 2ac9365d-b507-467e-85dd-54ce00dda7a5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:11:50Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-10T19:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:50Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c10370e82dab31 namespace: kserve-ci-e2e-test resourceVersion: "16282" uid: fb4082ed-9da5-4d83-8df4-9d0b8409b119 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:11:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:11:50Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c10370e82e9217 namespace: kserve-ci-e2e-test resourceVersion: "16283" uid: 316c789a-5b12-4aae-98cd-748b7b5a5853 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-10T19:11:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 6fcb7138-efd4-4092-b246-8f1726f843c7 kind: Event lastTimestamp: "2026-07-10T19:12:02Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-10T19:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:12:02Z" name: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l.18c1037143c1d06c namespace: kserve-ci-e2e-test resourceVersion: "16426" uid: e1cd0e48-9c07-4dd6-8488-ae66630336dc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-predictor-d68dc49bd namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: 70bed6e5-5e3f-4f4e-99c6-f0c56bea8c00 kind: Event lastTimestamp: "2026-07-10T19:10:05Z" message: 'Created pod: isvc-sklearn-batcher-predictor-d68dc49bd-n2f9l' metadata: creationTimestamp: "2026-07-10T19:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:05Z" name: isvc-sklearn-batcher-predictor-d68dc49bd.18c103586f04859b namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 0df29df6-4b91-4e1a-96a6-06fb36d6fe5b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15443" uid: a31be034-3c0a-4375-9bf4-a34220a48647 kind: Event lastTimestamp: "2026-07-10T19:10:05Z" message: Scaled up replica set isvc-sklearn-batcher-predictor-d68dc49bd from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:10:05Z" name: isvc-sklearn-batcher-predictor.18c103586982d4ea namespace: kserve-ci-e2e-test resourceVersion: "15446" uid: 2242021b-8043-4673-8465-e28b322f7a2a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:10:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15453" uid: 510462a9-b6d9-4f15-a533-ddd9f62dfa17 kind: Event lastTimestamp: "2026-07-10T19:11:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:05Z" name: isvc-sklearn-batcher-predictor.18c1035bee98e461 namespace: kserve-ci-e2e-test resourceVersion: "15928" uid: bfedef93-2119-411d-9156-666728097e7b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-10T19:10:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15453" uid: 510462a9-b6d9-4f15-a533-ddd9f62dfa17 kind: Event lastTimestamp: "2026-07-10T19:11:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:05Z" name: isvc-sklearn-batcher-predictor.18c1035beef762a0 namespace: kserve-ci-e2e-test resourceVersion: "15929" uid: 1d419e16-b8ba-44a6-8f3b-1f7337657e3a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:11:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15592" uid: 510462a9-b6d9-4f15-a533-ddd9f62dfa17 kind: Event lastTimestamp: "2026-07-10T19:11:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:11:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:35Z" name: isvc-sklearn-batcher-predictor.18c10369eaeee9ae namespace: kserve-ci-e2e-test resourceVersion: "16145" uid: 458c87f6-dc9f-41a7-a5d3-e8030bbf8d49 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:11:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "15592" uid: 510462a9-b6d9-4f15-a533-ddd9f62dfa17 kind: Event lastTimestamp: "2026-07-10T19:11:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-10T19:11:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:11:35Z" name: isvc-sklearn-batcher-predictor.18c10369eb502378 namespace: kserve-ci-e2e-test resourceVersion: "16146" uid: ac7cfbd8-5270-47bd-ae17-a015e6aa0e2d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-10T19:10:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: 19cffc99-f9b8-4a86-9a2a-51038adfc016 kind: Event lastTimestamp: "2026-07-10T19:11:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:43Z" name: isvc-sklearn-batcher.18c103586fa83a2c namespace: kserve-ci-e2e-test resourceVersion: "16208" uid: a43857a8-5920-4cfc-b43b-65d1ba415e6b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: 19cffc99-f9b8-4a86-9a2a-51038adfc016 kind: Event lastTimestamp: "2026-07-10T19:10:05Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:10:05Z" name: isvc-sklearn-batcher.18c103587000cdcc namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: b3be31be-7f1b-4f57-9d51-2693c551aec5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:10:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: 19cffc99-f9b8-4a86-9a2a-51038adfc016 kind: Event lastTimestamp: "2026-07-10T19:10:05Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:10:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:10:05Z" name: isvc-sklearn-batcher.18c1035870010a7a namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: 9e71c75b-9430-40ad-a5b3-85874760975a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:11:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "16205" uid: 19cffc99-f9b8-4a86-9a2a-51038adfc016 kind: Event lastTimestamp: "2026-07-10T19:11:43Z" message: InferenceService [isvc-sklearn-batcher] is Ready metadata: creationTimestamp: "2026-07-10T19:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:11:43Z" name: isvc-sklearn-batcher.18c1036f329b6ac6 namespace: kserve-ci-e2e-test resourceVersion: "16207" uid: 0b45eee3-858e-4e64-981a-64939c44dc0c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-10T19:13:26.187618Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-828h5 namespace: kserve-ci-e2e-test resourceVersion: "17082" uid: d4bd1df1-ea5a-489a-aaf1-cdd6ac3b295f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-predictor-6575bcfdfd-828h5 to ip-10-0-136-15.ec2.internal metadata: creationTimestamp: "2026-07-10T19:13:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-10T19:13:26Z" name: message-dumper-predictor-6575bcfdfd-828h5.18c1038718d6dcb0 namespace: kserve-ci-e2e-test resourceVersion: "17088" uid: d592b1a5-6556-4515-bfdf-80c6c28ee527 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5495ccf5fb-v5dkr source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:26Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-828h5 namespace: kserve-ci-e2e-test resourceVersion: "17084" uid: d4bd1df1-ea5a-489a-aaf1-cdd6ac3b295f kind: Event lastTimestamp: "2026-07-10T19:13:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-10T19:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:26Z" name: message-dumper-predictor-6575bcfdfd-828h5.18c1038722fa89f2 namespace: kserve-ci-e2e-test resourceVersion: "17097" uid: cbdf9189-2f57-4e07-96b0-9af5d804f13a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:27Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-828h5 namespace: kserve-ci-e2e-test resourceVersion: "17092" uid: d4bd1df1-ea5a-489a-aaf1-cdd6ac3b295f kind: Event lastTimestamp: "2026-07-10T19:13:27Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-10T19:13:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-10T19:13:27Z" name: message-dumper-predictor-6575bcfdfd-828h5.18c1038755f60a7f namespace: kserve-ci-e2e-test resourceVersion: "17103" uid: 77473167-d7be-438f-9852-f05e4660f2a4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-828h5 namespace: kserve-ci-e2e-test resourceVersion: "17084" uid: d4bd1df1-ea5a-489a-aaf1-cdd6ac3b295f kind: Event lastTimestamp: "2026-07-10T19:13:27Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-10T19:13:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:27Z" name: message-dumper-predictor-6575bcfdfd-828h5.18c1038757716f3b namespace: kserve-ci-e2e-test resourceVersion: "17105" uid: a617019e-7996-4311-9a67-46800a48f1df reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-828h5 namespace: kserve-ci-e2e-test resourceVersion: "17084" uid: d4bd1df1-ea5a-489a-aaf1-cdd6ac3b295f kind: Event lastTimestamp: "2026-07-10T19:13:28Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.246s (1.246s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-10T19:13:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:28Z" name: message-dumper-predictor-6575bcfdfd-828h5.18c10387a1b9e7cb namespace: kserve-ci-e2e-test resourceVersion: "17116" uid: e8cfcbe3-6da0-41c0-a865-76e4e2a3abf8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-828h5 namespace: kserve-ci-e2e-test resourceVersion: "17084" uid: d4bd1df1-ea5a-489a-aaf1-cdd6ac3b295f kind: Event lastTimestamp: "2026-07-10T19:13:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-10T19:13:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:28Z" name: message-dumper-predictor-6575bcfdfd-828h5.18c10387a487b727 namespace: kserve-ci-e2e-test resourceVersion: "17117" uid: 569b2bf4-1b85-421f-90a2-a0dbcbf31e3a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-828h5 namespace: kserve-ci-e2e-test resourceVersion: "17084" uid: d4bd1df1-ea5a-489a-aaf1-cdd6ac3b295f kind: Event lastTimestamp: "2026-07-10T19:13:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-10T19:13:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:28Z" name: message-dumper-predictor-6575bcfdfd-828h5.18c10387a4e635d2 namespace: kserve-ci-e2e-test resourceVersion: "17118" uid: 91da834b-4847-49e4-afaf-510618634c36 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-828h5 namespace: kserve-ci-e2e-test resourceVersion: "17084" uid: d4bd1df1-ea5a-489a-aaf1-cdd6ac3b295f kind: Event lastTimestamp: "2026-07-10T19:13:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-10T19:13:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:28Z" name: message-dumper-predictor-6575bcfdfd-828h5.18c10387a4eff5b5 namespace: kserve-ci-e2e-test resourceVersion: "17119" uid: e7946a2f-7bf2-42ff-ae12-134e46caec98 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-828h5 namespace: kserve-ci-e2e-test resourceVersion: "17084" uid: d4bd1df1-ea5a-489a-aaf1-cdd6ac3b295f kind: Event lastTimestamp: "2026-07-10T19:13:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-10T19:13:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:28Z" name: message-dumper-predictor-6575bcfdfd-828h5.18c10387a75b2cc1 namespace: kserve-ci-e2e-test resourceVersion: "17120" uid: 3129646a-9d0d-42e7-959b-b9b675ce44b0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-828h5 namespace: kserve-ci-e2e-test resourceVersion: "17084" uid: d4bd1df1-ea5a-489a-aaf1-cdd6ac3b295f kind: Event lastTimestamp: "2026-07-10T19:13:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:13:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:13:28Z" name: message-dumper-predictor-6575bcfdfd-828h5.18c10387a7b9d3f0 namespace: kserve-ci-e2e-test resourceVersion: "17121" uid: f271820e-33a8-4af9-adff-ce7903194971 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-828h5 namespace: kserve-ci-e2e-test resourceVersion: "17084" uid: d4bd1df1-ea5a-489a-aaf1-cdd6ac3b295f kind: Event lastTimestamp: "2026-07-10T19:15:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-10T19:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:21Z" name: message-dumper-predictor-6575bcfdfd-828h5.18c103a1f4053fb7 namespace: kserve-ci-e2e-test resourceVersion: "18072" uid: e052dae7-9e38-42cd-a234-64dd3b650ade reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:15:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-828h5 namespace: kserve-ci-e2e-test resourceVersion: "17084" uid: d4bd1df1-ea5a-489a-aaf1-cdd6ac3b295f kind: Event lastTimestamp: "2026-07-10T19:15:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-10T19:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-10T19:15:21Z" name: message-dumper-predictor-6575bcfdfd-828h5.18c103a1f405b3e5 namespace: kserve-ci-e2e-test resourceVersion: "18074" uid: d0add42f-5e9d-42bc-ab53-a5f00d32ed27 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-15.ec2.internal source: component: kubelet host: ip-10-0-136-15.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-predictor-6575bcfdfd namespace: kserve-ci-e2e-test resourceVersion: "17072" uid: c6684c5c-30e3-45d8-8d5b-eddd1750a4c2 kind: Event lastTimestamp: "2026-07-10T19:13:26Z" message: 'Created pod: message-dumper-predictor-6575bcfdfd-828h5' metadata: creationTimestamp: "2026-07-10T19:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:13:26Z" name: message-dumper-predictor-6575bcfdfd.18c10387184490c1 namespace: kserve-ci-e2e-test resourceVersion: "17083" uid: ae01a5d0-7b3f-44c9-be52-2ae6e47c8e83 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "17068" uid: 6230b685-43fb-48bc-98c8-42c1c13cbaeb kind: Event lastTimestamp: "2026-07-10T19:13:26Z" message: Scaled up replica set message-dumper-predictor-6575bcfdfd from 0 to 1 metadata: creationTimestamp: "2026-07-10T19:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:13:26Z" name: message-dumper-predictor.18c1038714d86c04 namespace: kserve-ci-e2e-test resourceVersion: "17073" uid: 677316be-048c-4e83-a512-b80ada632f20 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:13:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "17071" uid: 56182212-4a88-4a79-80d5-65cd299d8913 kind: Event lastTimestamp: "2026-07-10T19:13:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:13:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:13:56Z" name: message-dumper-predictor.18c1038a93b305e7 namespace: kserve-ci-e2e-test resourceVersion: "17396" uid: d31ccf68-ba43-4ab4-8a80-901c33f26054 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-10T19:13:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "17071" uid: 56182212-4a88-4a79-80d5-65cd299d8913 kind: Event lastTimestamp: "2026-07-10T19:13:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-10T19:13:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-10T19:13:56Z" name: message-dumper-predictor.18c1038a941ba560 namespace: kserve-ci-e2e-test resourceVersion: "17397" uid: ccb7fea7-0cf0-4f0f-94e4-96624db8a654 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-10T19:13:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "17044" uid: 5c992353-9fcb-4a02-ac4d-06d808bd6fe9 kind: Event lastTimestamp: "2026-07-10T19:13:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-10T19:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:13:37Z" name: message-dumper.18c10387151ae163 namespace: kserve-ci-e2e-test resourceVersion: "17193" uid: bad50470-286a-4e53-9667-461f1646c35f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "17044" uid: 5c992353-9fcb-4a02-ac4d-06d808bd6fe9 kind: Event lastTimestamp: "2026-07-10T19:13:26Z" message: 'Failed to update status for InferenceService "message-dumper": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-10T19:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:13:26Z" name: message-dumper.18c103871598ba7d namespace: kserve-ci-e2e-test resourceVersion: "17077" uid: a11c83cc-3296-4f65-a6a5-845b88fb6306 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-10T19:13:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "17191" uid: 5c992353-9fcb-4a02-ac4d-06d808bd6fe9 kind: Event lastTimestamp: "2026-07-10T19:13:37Z" message: InferenceService [message-dumper] is Ready metadata: creationTimestamp: "2026-07-10T19:13:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-10T19:13:37Z" name: message-dumper.18c10389ac31dda4 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: 8a2ae184-b860-4d50-b4a1-aa49b88c3ba1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22961"