--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:58:59.758468Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm namespace: kserve-ci-e2e-test resourceVersion: "21324" uid: 9ea1c4d5-4681-4fc6-9311-97ef52cd8f64 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-07-08T16:58:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:58:59Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm.18c05f07d993e6f8 namespace: kserve-ci-e2e-test resourceVersion: "21329" uid: c7d3b1c0-c936-414d-8044-b67850c0dbae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-867dc68758-8x97b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:00Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: 9ea1c4d5-4681-4fc6-9311-97ef52cd8f64 kind: Event lastTimestamp: "2026-07-08T16:59:00Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:59:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:59:00Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm.18c05f07f35ff287 namespace: kserve-ci-e2e-test resourceVersion: "21341" uid: 3bcb0ed1-2b6f-41cc-a15c-dfecb5198ed7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 9ea1c4d5-4681-4fc6-9311-97ef52cd8f64 kind: Event lastTimestamp: "2026-07-08T16:59:00Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:43b18792ad3c8c70fe63e50dee47838dba7fd6b00a2dcfe15c76c1dafc2c7ad8" already present on machine metadata: creationTimestamp: "2026-07-08T16:59:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:00Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm.18c05f07f4cda554 namespace: kserve-ci-e2e-test resourceVersion: "21343" uid: 29b28d19-e5bc-48c1-93e3-6cb92404509a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 9ea1c4d5-4681-4fc6-9311-97ef52cd8f64 kind: Event lastTimestamp: "2026-07-08T16:59:00Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:59:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:00Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm.18c05f07f7ad919c namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 624403f8-f30b-44af-a4d3-61989f94d44b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 9ea1c4d5-4681-4fc6-9311-97ef52cd8f64 kind: Event lastTimestamp: "2026-07-08T16:59:00Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:59:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:00Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm.18c05f07f8265fde namespace: kserve-ci-e2e-test resourceVersion: "21345" uid: 12c6d8e0-f22b-4514-8abe-33656aacc9f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 9ea1c4d5-4681-4fc6-9311-97ef52cd8f64 kind: Event lastTimestamp: "2026-07-08T16:59:09Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-08T16:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:09Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm.18c05f0a019f6c55 namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: 3a404810-16e6-4606-b6a8-f7ce35851a72 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 9ea1c4d5-4681-4fc6-9311-97ef52cd8f64 kind: Event lastTimestamp: "2026-07-08T16:59:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:09Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm.18c05f0a053b4d85 namespace: kserve-ci-e2e-test resourceVersion: "21416" uid: 04e1a7fb-4ba8-4f32-8ec5-dcfe3b686579 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 9ea1c4d5-4681-4fc6-9311-97ef52cd8f64 kind: Event lastTimestamp: "2026-07-08T16:59:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:09Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm.18c05f0a05b3ab90 namespace: kserve-ci-e2e-test resourceVersion: "21417" uid: 3eb1dfa5-a1a1-429c-80b2-7abe6499e39b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 9ea1c4d5-4681-4fc6-9311-97ef52cd8f64 kind: Event lastTimestamp: "2026-07-08T16:59:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:09Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm.18c05f0a05bdfefa namespace: kserve-ci-e2e-test resourceVersion: "21419" uid: 841b6f32-fb1f-4b74-832c-864c15d4608a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 9ea1c4d5-4681-4fc6-9311-97ef52cd8f64 kind: Event lastTimestamp: "2026-07-08T16:59:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:09Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm.18c05f0a08c79fcd namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: 3580298d-8795-449c-986d-ec4bf6a42d54 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 9ea1c4d5-4681-4fc6-9311-97ef52cd8f64 kind: Event lastTimestamp: "2026-07-08T16:59:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:09Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm.18c05f0a09428a94 namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: 2b6153b0-c18e-45e3-9d8e-76f85026c479 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:59:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 9ea1c4d5-4681-4fc6-9311-97ef52cd8f64 kind: Event lastTimestamp: "2026-07-08T16:59:16Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:16Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm.18c05f0a3da7b909 namespace: kserve-ci-e2e-test resourceVersion: "21479" uid: 5562273d-f3da-4d26-91b2-16bb43eb5f54 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 9ea1c4d5-4681-4fc6-9311-97ef52cd8f64 kind: Event lastTimestamp: "2026-07-08T16:59:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:29Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm.18c05f0ed7c43c4a namespace: kserve-ci-e2e-test resourceVersion: "21595" uid: 8004e43a-4063-4c95-94fa-fb1922ca0d25 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 9ea1c4d5-4681-4fc6-9311-97ef52cd8f64 kind: Event lastTimestamp: "2026-07-08T16:59:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:29Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm.18c05f0ed7c4bf72 namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: 3636aab7-062a-4e6c-bb3f-4f48dd9e4ee7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-ts-v1-predictor-89d67dbf7 namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: 569a3afc-fecb-4961-a645-35b8733a1fba kind: Event lastTimestamp: "2026-07-08T16:58:59Z" message: 'Created pod: isvc-autogluon-ts-v1-predictor-89d67dbf7-txkbm' metadata: creationTimestamp: "2026-07-08T16:58:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:59Z" name: isvc-autogluon-ts-v1-predictor-89d67dbf7.18c05f07d8e40dc6 namespace: kserve-ci-e2e-test resourceVersion: "21325" uid: 6d3655fd-435e-4085-87c5-b63e4f304949 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: 1445cfa5-dfca-49b9-9c04-a510aa55a394 kind: Event lastTimestamp: "2026-07-08T16:58:59Z" message: Scaled up replica set isvc-autogluon-ts-v1-predictor-89d67dbf7 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:58:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:59Z" name: isvc-autogluon-ts-v1-predictor.18c05f07d442034e namespace: kserve-ci-e2e-test resourceVersion: "21316" uid: 40a231cf-2e7f-4aa2-9556-11ef55d154e6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:59:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21314" uid: 804e295b-9f11-4b8e-a4b6-e6f10212706c kind: Event lastTimestamp: "2026-07-08T16:59:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:29Z" name: isvc-autogluon-ts-v1-predictor.18c05f0b5362a89a namespace: kserve-ci-e2e-test resourceVersion: "21580" uid: e03060f7-ac6f-4205-b7ea-d58a87b9182e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:59:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-ts-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "21314" uid: 804e295b-9f11-4b8e-a4b6-e6f10212706c kind: Event lastTimestamp: "2026-07-08T16:59:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:59:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:29Z" name: isvc-autogluon-ts-v1-predictor.18c05f0b53d1d44a namespace: kserve-ci-e2e-test resourceVersion: "21581" uid: 4962223c-b3a6-42a4-a1cb-2990323e1811 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-08T16:58:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21306" uid: 585f5060-36ea-40ed-b199-b06adfc34c2a kind: Event lastTimestamp: "2026-07-08T16:59:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:58:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:59:26Z" name: isvc-autogluon-ts-v1.18c05f07d457b40b namespace: kserve-ci-e2e-test resourceVersion: "21552" uid: 06672699-8b0c-43a0-9770-231a3bf4439e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21306" uid: 585f5060-36ea-40ed-b199-b06adfc34c2a kind: Event lastTimestamp: "2026-07-08T16:58:59Z" message: 'Failed to update status for InferenceService "isvc-autogluon-ts-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:58:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:58:59Z" name: isvc-autogluon-ts-v1.18c05f07d4c218fc namespace: kserve-ci-e2e-test resourceVersion: "21319" uid: d990836d-4fda-42f1-9bee-15a9504a23d8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21306" uid: 585f5060-36ea-40ed-b199-b06adfc34c2a kind: Event lastTimestamp: "2026-07-08T16:58:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-ts-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:58:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:58:59Z" name: isvc-autogluon-ts-v1.18c05f07d4c261e1 namespace: kserve-ci-e2e-test resourceVersion: "21320" uid: 37400df6-0bc5-4034-9176-09afc28ac66e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-ts-v1 namespace: kserve-ci-e2e-test resourceVersion: "21550" uid: 585f5060-36ea-40ed-b199-b06adfc34c2a kind: Event lastTimestamp: "2026-07-08T16:59:26Z" message: InferenceService [isvc-autogluon-ts-v1] is Ready metadata: creationTimestamp: "2026-07-08T16:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:59:26Z" name: isvc-autogluon-ts-v1.18c05f0dfd9d35a6 namespace: kserve-ci-e2e-test resourceVersion: "21551" uid: 79d7b920-f0a4-4c16-8f7e-b9d141852a0b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:50:08.154973Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: 5271cd43-db59-417f-88ec-336654783e7b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-07-08T16:50:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:50:08Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q.18c05e8c138a4b6f namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 1509f42d-ffa5-4aa8-b9d6-6f3f80419fd1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-867dc68758-8x97b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:08Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q namespace: kserve-ci-e2e-test resourceVersion: "17289" uid: 5271cd43-db59-417f-88ec-336654783e7b kind: Event lastTimestamp: "2026-07-08T16:50:08Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:50:08Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q.18c05e8c32455fc7 namespace: kserve-ci-e2e-test resourceVersion: "17306" uid: 186e8ba3-5214-4382-a6d6-4ec463f6b803 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: 5271cd43-db59-417f-88ec-336654783e7b kind: Event lastTimestamp: "2026-07-08T16:50:08Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:43b18792ad3c8c70fe63e50dee47838dba7fd6b00a2dcfe15c76c1dafc2c7ad8" already present on machine metadata: creationTimestamp: "2026-07-08T16:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:08Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q.18c05e8c33b1dd2d namespace: kserve-ci-e2e-test resourceVersion: "17309" uid: 2f2c97e9-d9dc-4170-a58d-0a596e4def02 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: 5271cd43-db59-417f-88ec-336654783e7b kind: Event lastTimestamp: "2026-07-08T16:50:08Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:08Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q.18c05e8c36af3b56 namespace: kserve-ci-e2e-test resourceVersion: "17310" uid: d2b1f609-3c7f-4544-9e2d-81f379a5d4dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: 5271cd43-db59-417f-88ec-336654783e7b kind: Event lastTimestamp: "2026-07-08T16:50:08Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:08Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q.18c05e8c3727cab8 namespace: kserve-ci-e2e-test resourceVersion: "17311" uid: 3bc7684a-a9a8-4c65-ae1d-1fe8fa936f62 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: 5271cd43-db59-417f-88ec-336654783e7b kind: Event lastTimestamp: "2026-07-08T16:50:13Z" message: Pulling image "kserve/autogluonserver:latest" metadata: creationTimestamp: "2026-07-08T16:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:13Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q.18c05e8d536ccdfe namespace: kserve-ci-e2e-test resourceVersion: "17349" uid: 4440a3e3-ed92-42a8-a29a-4e0a54deab41 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: 5271cd43-db59-417f-88ec-336654783e7b kind: Event lastTimestamp: "2026-07-08T16:52:22Z" message: 'Successfully pulled image "kserve/autogluonserver:latest" in 2m8.869s (2m8.869s including waiting). Image size: 8939170587 bytes.' metadata: creationTimestamp: "2026-07-08T16:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:52:22Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q.18c05eab54a73a70 namespace: kserve-ci-e2e-test resourceVersion: "18258" uid: 3f0bf324-756c-4790-ad11-39e0a8ec9b21 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: 5271cd43-db59-417f-88ec-336654783e7b kind: Event lastTimestamp: "2026-07-08T16:52:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:52:22Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q.18c05eab59cf45a0 namespace: kserve-ci-e2e-test resourceVersion: "18259" uid: fdad439e-3694-4c3b-85f5-97bf16264832 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: 5271cd43-db59-417f-88ec-336654783e7b kind: Event lastTimestamp: "2026-07-08T16:52:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:52:22Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q.18c05eab5ae69641 namespace: kserve-ci-e2e-test resourceVersion: "18260" uid: 72f8b838-9efe-4c0f-9685-26dfe21adc47 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: 5271cd43-db59-417f-88ec-336654783e7b kind: Event lastTimestamp: "2026-07-08T16:52:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:52:22Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q.18c05eab5af3d1aa namespace: kserve-ci-e2e-test resourceVersion: "18261" uid: 043c1df5-a07a-4b3b-8985-bd9b64d71878 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: 5271cd43-db59-417f-88ec-336654783e7b kind: Event lastTimestamp: "2026-07-08T16:52:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:52:22Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q.18c05eab5fb3bf43 namespace: kserve-ci-e2e-test resourceVersion: "18262" uid: ffdbd5d2-bc8f-4b7f-bc74-66f159d0eeaa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: 5271cd43-db59-417f-88ec-336654783e7b kind: Event lastTimestamp: "2026-07-08T16:52:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:52:22Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q.18c05eab60c12041 namespace: kserve-ci-e2e-test resourceVersion: "18263" uid: f5b0b361-cad1-4a1a-bc81-d733543d8d0f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:52:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: 5271cd43-db59-417f-88ec-336654783e7b kind: Event lastTimestamp: "2026-07-08T16:52:49Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:52:49Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q.18c05eab843a1d54 namespace: kserve-ci-e2e-test resourceVersion: "18480" uid: 40d58f15-9b20-48b1-9cab-c2f7bef4e906 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: 5271cd43-db59-417f-88ec-336654783e7b kind: Event lastTimestamp: "2026-07-08T16:52:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:52:48Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q.18c05eb15d8cd0d1 namespace: kserve-ci-e2e-test resourceVersion: "18471" uid: 174dcb86-4511-41bb-8b8f-026d82f158e9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: 5271cd43-db59-417f-88ec-336654783e7b kind: Event lastTimestamp: "2026-07-08T16:52:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:52:48Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q.18c05eb15d8d2b72 namespace: kserve-ci-e2e-test resourceVersion: "18472" uid: 239364d2-0ecf-4b83-9574-d5fa529e3d60 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: 5271cd43-db59-417f-88ec-336654783e7b kind: Event lastTimestamp: "2026-07-08T16:52:49Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:52:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:52:49Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q.18c05eb192446284 namespace: kserve-ci-e2e-test resourceVersion: "18479" uid: 636dfce6-4abe-4310-891e-c7b41d5ec47f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v1-predictor-7bbbbd79cd namespace: kserve-ci-e2e-test resourceVersion: "17247" uid: 13252cd4-65ec-4879-8960-0eaaea397ee8 kind: Event lastTimestamp: "2026-07-08T16:50:08Z" message: 'Created pod: isvc-autogluon-v1-predictor-7bbbbd79cd-l6s6q' metadata: creationTimestamp: "2026-07-08T16:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:50:08Z" name: isvc-autogluon-v1-predictor-7bbbbd79cd.18c05e8c13210b16 namespace: kserve-ci-e2e-test resourceVersion: "17280" uid: c18424ce-a896-4c52-b548-7dd0ac8b605a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17240" uid: 27de8da7-b3e9-49fd-ba62-60ef850ef5a1 kind: Event lastTimestamp: "2026-07-08T16:50:08Z" message: Scaled up replica set isvc-autogluon-v1-predictor-7bbbbd79cd from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:50:08Z" name: isvc-autogluon-v1-predictor.18c05e8c0ddd017c namespace: kserve-ci-e2e-test resourceVersion: "17252" uid: 093603cd-acbb-4b60-b902-fbc991b00968 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T16:50:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17246" uid: bb3ca4fd-5a37-449c-864a-370dd3a38294 kind: Event lastTimestamp: "2026-07-08T16:52:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:50:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:52:38Z" name: isvc-autogluon-v1-predictor.18c05e8f8c66cf18 namespace: kserve-ci-e2e-test resourceVersion: "18373" uid: e5d70519-21d7-45c8-87a4-c39a38cb6078 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T16:50:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v1-predictor namespace: kserve-ci-e2e-test resourceVersion: "17246" uid: bb3ca4fd-5a37-449c-864a-370dd3a38294 kind: Event lastTimestamp: "2026-07-08T16:52:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:50:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:52:38Z" name: isvc-autogluon-v1-predictor.18c05e8f8ce514a9 namespace: kserve-ci-e2e-test resourceVersion: "18374" uid: 18e1d01b-7e6d-47df-bae5-be913a8042d5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-08T16:50:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17228" uid: 49e5077d-b23b-44c7-96dc-aafd515b34d3 kind: Event lastTimestamp: "2026-07-08T16:52:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:52:39Z" name: isvc-autogluon-v1.18c05e8c0e15087c namespace: kserve-ci-e2e-test resourceVersion: "18392" uid: db0315d0-8f6d-4ca9-903a-c605cb1f3a45 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:50:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17228" uid: 49e5077d-b23b-44c7-96dc-aafd515b34d3 kind: Event lastTimestamp: "2026-07-08T16:50:08Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:50:08Z" name: isvc-autogluon-v1.18c05e8c0e7ffbc1 namespace: kserve-ci-e2e-test resourceVersion: "17293" uid: b23e6507-5e35-4d8e-9074-7929cd6258d4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:50:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "17228" uid: 49e5077d-b23b-44c7-96dc-aafd515b34d3 kind: Event lastTimestamp: "2026-07-08T16:50:08Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:50:08Z" name: isvc-autogluon-v1.18c05e8c0e8061d8 namespace: kserve-ci-e2e-test resourceVersion: "17295" uid: 9d3eb16f-18b0-493d-a133-3cf0545bbdb5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v1 namespace: kserve-ci-e2e-test resourceVersion: "18390" uid: 49e5077d-b23b-44c7-96dc-aafd515b34d3 kind: Event lastTimestamp: "2026-07-08T16:52:39Z" message: InferenceService [isvc-autogluon-v1] is Ready metadata: creationTimestamp: "2026-07-08T16:52:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:52:39Z" name: isvc-autogluon-v1.18c05eaf43c1abfe namespace: kserve-ci-e2e-test resourceVersion: "18391" uid: 55380c46-91fe-4b6c-a36d-2faf9bf25e46 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:56:44.391195Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp namespace: kserve-ci-e2e-test resourceVersion: "20301" uid: 733ffc72-f62e-4330-afdc-bc7b2464dd39 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-07-08T16:56:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:56:44Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp.18c05ee8550f3f60 namespace: kserve-ci-e2e-test resourceVersion: "20306" uid: 8032ba24-ac3b-437a-8dfe-37a3f1dd0d70 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-867dc68758-8x97b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:44Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: 733ffc72-f62e-4330-afdc-bc7b2464dd39 kind: Event lastTimestamp: "2026-07-08T16:56:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-autogluon-v2-noslash-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:56:44Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp.18c05ee8622e4a43 namespace: kserve-ci-e2e-test resourceVersion: "20319" uid: 5dd52271-88c1-4b82-9dee-79ea4dbb038e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp namespace: kserve-ci-e2e-test resourceVersion: "20311" uid: 733ffc72-f62e-4330-afdc-bc7b2464dd39 kind: Event lastTimestamp: "2026-07-08T16:56:45Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:56:45Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp.18c05ee892e0197d namespace: kserve-ci-e2e-test resourceVersion: "20326" uid: d9500367-712b-4524-ba15-056c2519c78b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: 733ffc72-f62e-4330-afdc-bc7b2464dd39 kind: Event lastTimestamp: "2026-07-08T16:56:45Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:43b18792ad3c8c70fe63e50dee47838dba7fd6b00a2dcfe15c76c1dafc2c7ad8" already present on machine metadata: creationTimestamp: "2026-07-08T16:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:56:45Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp.18c05ee8958f8830 namespace: kserve-ci-e2e-test resourceVersion: "20328" uid: 9657ab08-97e1-479e-8434-5a6c7a622e17 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: 733ffc72-f62e-4330-afdc-bc7b2464dd39 kind: Event lastTimestamp: "2026-07-08T16:56:45Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:56:45Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp.18c05ee899058679 namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 2392e7be-8211-4b7e-b529-07197c734e5c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: 733ffc72-f62e-4330-afdc-bc7b2464dd39 kind: Event lastTimestamp: "2026-07-08T16:56:45Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:56:45Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp.18c05ee89983f93b namespace: kserve-ci-e2e-test resourceVersion: "20331" uid: a14b1623-4ed2-4d0b-ae17-f7870e89a903 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: 733ffc72-f62e-4330-afdc-bc7b2464dd39 kind: Event lastTimestamp: "2026-07-08T16:57:07Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-08T16:57:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:07Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp.18c05eedb4853786 namespace: kserve-ci-e2e-test resourceVersion: "20489" uid: e373b516-35dd-42b1-9715-9c2023544726 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: 733ffc72-f62e-4330-afdc-bc7b2464dd39 kind: Event lastTimestamp: "2026-07-08T16:57:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:57:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:07Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp.18c05eedb7d694e9 namespace: kserve-ci-e2e-test resourceVersion: "20494" uid: d97ac3c5-799c-4e7b-949a-2fddf362f919 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: 733ffc72-f62e-4330-afdc-bc7b2464dd39 kind: Event lastTimestamp: "2026-07-08T16:57:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:57:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:07Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp.18c05eedb85f2b7f namespace: kserve-ci-e2e-test resourceVersion: "20495" uid: e0115bd0-53c7-4c17-8b8b-64f491fbced8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: 733ffc72-f62e-4330-afdc-bc7b2464dd39 kind: Event lastTimestamp: "2026-07-08T16:57:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:57:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:07Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp.18c05eedb86aaafd namespace: kserve-ci-e2e-test resourceVersion: "20496" uid: 33f188d9-6d62-481e-b3dc-68704a797ad3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: 733ffc72-f62e-4330-afdc-bc7b2464dd39 kind: Event lastTimestamp: "2026-07-08T16:57:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:57:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:07Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp.18c05eedbc4450e1 namespace: kserve-ci-e2e-test resourceVersion: "20498" uid: a146869d-a8a1-458b-be07-5c274d09815d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: 733ffc72-f62e-4330-afdc-bc7b2464dd39 kind: Event lastTimestamp: "2026-07-08T16:57:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:57:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:07Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp.18c05eedbcc05c8c namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: a06a396a-5981-4756-865b-9926e873f243 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: 733ffc72-f62e-4330-afdc-bc7b2464dd39 kind: Event lastTimestamp: "2026-07-08T16:58:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:58:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:54Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp.18c05f06a52d42ef namespace: kserve-ci-e2e-test resourceVersion: "21263" uid: 361324a9-ebce-444c-86d9-690f514d480e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: 733ffc72-f62e-4330-afdc-bc7b2464dd39 kind: Event lastTimestamp: "2026-07-08T16:58:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:58:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:58:54Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp.18c05f06a52ddcfa namespace: kserve-ci-e2e-test resourceVersion: "21264" uid: 96f9913c-55d7-41ba-a32c-49949db0a424 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84 namespace: kserve-ci-e2e-test resourceVersion: "20291" uid: 6aeb8714-92ed-4d57-b73f-58712dbec738 kind: Event lastTimestamp: "2026-07-08T16:56:44Z" message: 'Created pod: isvc-autogluon-v2-noslash-predictor-9dcfbbb84-677kp' metadata: creationTimestamp: "2026-07-08T16:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:44Z" name: isvc-autogluon-v2-noslash-predictor-9dcfbbb84.18c05ee854630d00 namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: 720937e2-7733-422a-8738-cd3ba6a1b479 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20289" uid: baf80a9d-e9ed-4a6b-8a86-2c5fe6c8aeb9 kind: Event lastTimestamp: "2026-07-08T16:56:44Z" message: Scaled up replica set isvc-autogluon-v2-noslash-predictor-9dcfbbb84 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:44Z" name: isvc-autogluon-v2-noslash-predictor.18c05ee84fa9810b namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 90dc0d6a-d450-499b-b1e1-461c0914ac2c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:56:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20290" uid: 740ed6ae-bb9e-48c1-a930-7e5829a4e149 kind: Event lastTimestamp: "2026-07-08T16:57:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:56:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:57:29Z" name: isvc-autogluon-v2-noslash-predictor.18c05eebce7b64ba namespace: kserve-ci-e2e-test resourceVersion: "20650" uid: 19025c2f-1bc7-439e-8071-fdada5dcac44 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:56:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20290" uid: 740ed6ae-bb9e-48c1-a930-7e5829a4e149 kind: Event lastTimestamp: "2026-07-08T16:57:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:56:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:57:29Z" name: isvc-autogluon-v2-noslash-predictor.18c05eebceeaa728 namespace: kserve-ci-e2e-test resourceVersion: "20651" uid: 71111ee3-367e-472a-9550-4060e3ae0ef8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:57:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20430" uid: 740ed6ae-bb9e-48c1-a930-7e5829a4e149 kind: Event lastTimestamp: "2026-07-08T16:58:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:44Z" name: isvc-autogluon-v2-noslash-predictor.18c05ef64d6f0b53 namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: 47a40971-01b7-4894-977f-a9e28ef22a88 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:57:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-noslash-predictor namespace: kserve-ci-e2e-test resourceVersion: "20430" uid: 740ed6ae-bb9e-48c1-a930-7e5829a4e149 kind: Event lastTimestamp: "2026-07-08T16:58:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:57:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:58:44Z" name: isvc-autogluon-v2-noslash-predictor.18c05ef64dddc02a namespace: kserve-ci-e2e-test resourceVersion: "21174" uid: c76db334-e60b-4ec3-9fd0-35573db00e19 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-08T16:56:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: f3c27848-9bb9-4726-9077-0ccce5ddb7df kind: Event lastTimestamp: "2026-07-08T16:58:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:58:44Z" name: isvc-autogluon-v2-noslash.18c05ee84fae69a3 namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: 87b8fa22-fc8b-45bf-a60f-c9d9bfd840a4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:56:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: f3c27848-9bb9-4726-9077-0ccce5ddb7df kind: Event lastTimestamp: "2026-07-08T16:56:44Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-noslash": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:56:44Z" name: isvc-autogluon-v2-noslash.18c05ee8503610eb namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 18fe4936-e513-4a99-80ac-da0907bd7b82 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:56:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: f3c27848-9bb9-4726-9077-0ccce5ddb7df kind: Event lastTimestamp: "2026-07-08T16:56:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-noslash": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:56:44Z" name: isvc-autogluon-v2-noslash.18c05ee85036562c namespace: kserve-ci-e2e-test resourceVersion: "20312" uid: f4dc8317-e85b-4fd9-a677-dc45cfec581b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:58:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-noslash namespace: kserve-ci-e2e-test resourceVersion: "21168" uid: f3c27848-9bb9-4726-9077-0ccce5ddb7df kind: Event lastTimestamp: "2026-07-08T16:58:44Z" message: InferenceService [isvc-autogluon-v2-noslash] is Ready metadata: creationTimestamp: "2026-07-08T16:58:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:58:44Z" name: isvc-autogluon-v2-noslash.18c05f044058917b namespace: kserve-ci-e2e-test resourceVersion: "21169" uid: 06361f0d-7332-4e31-89b0-396d684b4b4e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:52:53.713973Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-c4hzl namespace: kserve-ci-e2e-test resourceVersion: "18537" uid: a343106f-89e5-4fe2-965c-b1f5055699c5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-predictor-67b849d456-c4hzl to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-07-08T16:52:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:52:53Z" name: isvc-autogluon-v2-predictor-67b849d456-c4hzl.18c05eb29fa030c8 namespace: kserve-ci-e2e-test resourceVersion: "18541" uid: a8f9f767-210c-45e7-b4b1-3df3836e9247 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-867dc68758-8x97b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:54Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-c4hzl namespace: kserve-ci-e2e-test resourceVersion: "18551" uid: a343106f-89e5-4fe2-965c-b1f5055699c5 kind: Event lastTimestamp: "2026-07-08T16:52:54Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:52:54Z" name: isvc-autogluon-v2-predictor-67b849d456-c4hzl.18c05eb2bbe963d8 namespace: kserve-ci-e2e-test resourceVersion: "18557" uid: 5de3ce3e-fd9b-425e-bd2e-8dca1bb05d88 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-c4hzl namespace: kserve-ci-e2e-test resourceVersion: "18539" uid: a343106f-89e5-4fe2-965c-b1f5055699c5 kind: Event lastTimestamp: "2026-07-08T16:52:54Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:43b18792ad3c8c70fe63e50dee47838dba7fd6b00a2dcfe15c76c1dafc2c7ad8" already present on machine metadata: creationTimestamp: "2026-07-08T16:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:52:54Z" name: isvc-autogluon-v2-predictor-67b849d456-c4hzl.18c05eb2bda9acd6 namespace: kserve-ci-e2e-test resourceVersion: "18559" uid: 8b8ee737-242a-4219-bdfa-48b78e9940fc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-c4hzl namespace: kserve-ci-e2e-test resourceVersion: "18539" uid: a343106f-89e5-4fe2-965c-b1f5055699c5 kind: Event lastTimestamp: "2026-07-08T16:52:54Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:52:54Z" name: isvc-autogluon-v2-predictor-67b849d456-c4hzl.18c05eb2c4385291 namespace: kserve-ci-e2e-test resourceVersion: "18561" uid: 4e99a343-1189-4a8a-bd11-167dc94306c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-c4hzl namespace: kserve-ci-e2e-test resourceVersion: "18539" uid: a343106f-89e5-4fe2-965c-b1f5055699c5 kind: Event lastTimestamp: "2026-07-08T16:52:54Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:52:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:52:54Z" name: isvc-autogluon-v2-predictor-67b849d456-c4hzl.18c05eb2c56663f2 namespace: kserve-ci-e2e-test resourceVersion: "18562" uid: 2673d3d4-f6d5-44ad-8b81-bc33e0e5666e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-c4hzl namespace: kserve-ci-e2e-test resourceVersion: "18539" uid: a343106f-89e5-4fe2-965c-b1f5055699c5 kind: Event lastTimestamp: "2026-07-08T16:53:23Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-08T16:53:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:23Z" name: isvc-autogluon-v2-predictor-67b849d456-c4hzl.18c05eb98ce2e713 namespace: kserve-ci-e2e-test resourceVersion: "18765" uid: f34bcdbd-0b9b-411b-abaf-36c432deb517 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-c4hzl namespace: kserve-ci-e2e-test resourceVersion: "18539" uid: a343106f-89e5-4fe2-965c-b1f5055699c5 kind: Event lastTimestamp: "2026-07-08T16:53:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:53:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:23Z" name: isvc-autogluon-v2-predictor-67b849d456-c4hzl.18c05eb98faca9cd namespace: kserve-ci-e2e-test resourceVersion: "18768" uid: 8bed54cc-95f4-41b4-a20a-06301aedbf96 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-c4hzl namespace: kserve-ci-e2e-test resourceVersion: "18539" uid: a343106f-89e5-4fe2-965c-b1f5055699c5 kind: Event lastTimestamp: "2026-07-08T16:53:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:53:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:23Z" name: isvc-autogluon-v2-predictor-67b849d456-c4hzl.18c05eb9901769b3 namespace: kserve-ci-e2e-test resourceVersion: "18769" uid: a19c8722-63af-4355-ae8e-d0fb4754d9f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-c4hzl namespace: kserve-ci-e2e-test resourceVersion: "18539" uid: a343106f-89e5-4fe2-965c-b1f5055699c5 kind: Event lastTimestamp: "2026-07-08T16:53:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:53:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:23Z" name: isvc-autogluon-v2-predictor-67b849d456-c4hzl.18c05eb9902278a4 namespace: kserve-ci-e2e-test resourceVersion: "18772" uid: 1a915a75-11a6-4567-8b2e-47a9c255eec6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-c4hzl namespace: kserve-ci-e2e-test resourceVersion: "18539" uid: a343106f-89e5-4fe2-965c-b1f5055699c5 kind: Event lastTimestamp: "2026-07-08T16:53:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:53:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:23Z" name: isvc-autogluon-v2-predictor-67b849d456-c4hzl.18c05eb992fcdf72 namespace: kserve-ci-e2e-test resourceVersion: "18775" uid: b2d9289d-3b70-4ec1-bb78-f38a36a6ad11 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-c4hzl namespace: kserve-ci-e2e-test resourceVersion: "18539" uid: a343106f-89e5-4fe2-965c-b1f5055699c5 kind: Event lastTimestamp: "2026-07-08T16:53:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:53:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:23Z" name: isvc-autogluon-v2-predictor-67b849d456-c4hzl.18c05eb9936e57f9 namespace: kserve-ci-e2e-test resourceVersion: "18776" uid: e424dae5-b7b4-48fa-ba64-cf76aaa0e3e3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-c4hzl namespace: kserve-ci-e2e-test resourceVersion: "18539" uid: a343106f-89e5-4fe2-965c-b1f5055699c5 kind: Event lastTimestamp: "2026-07-08T16:54:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:53Z" name: isvc-autogluon-v2-predictor-67b849d456-c4hzl.18c05ece9a1d96dd namespace: kserve-ci-e2e-test resourceVersion: "19424" uid: 2be15df1-3bdc-4749-894d-bbe509f5d3cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-predictor-67b849d456-c4hzl namespace: kserve-ci-e2e-test resourceVersion: "18539" uid: a343106f-89e5-4fe2-965c-b1f5055699c5 kind: Event lastTimestamp: "2026-07-08T16:54:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:53Z" name: isvc-autogluon-v2-predictor-67b849d456-c4hzl.18c05ece9a1e3d28 namespace: kserve-ci-e2e-test resourceVersion: "19425" uid: c45df65a-05ab-4d01-b97a-1b3fbfb7deca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-predictor-67b849d456 namespace: kserve-ci-e2e-test resourceVersion: "18527" uid: 290cf40d-900d-4431-ba28-abfa6ec82040 kind: Event lastTimestamp: "2026-07-08T16:52:53Z" message: 'Created pod: isvc-autogluon-v2-predictor-67b849d456-c4hzl' metadata: creationTimestamp: "2026-07-08T16:52:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:52:53Z" name: isvc-autogluon-v2-predictor-67b849d456.18c05eb29f2b9fcb namespace: kserve-ci-e2e-test resourceVersion: "18538" uid: d7d410b7-5cc0-479a-8867-98dacb7d5f83 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18524" uid: 1895566b-e7f9-401c-a3d9-31b757f34c86 kind: Event lastTimestamp: "2026-07-08T16:52:53Z" message: Scaled up replica set isvc-autogluon-v2-predictor-67b849d456 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:52:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:52:53Z" name: isvc-autogluon-v2-predictor.18c05eb29af47085 namespace: kserve-ci-e2e-test resourceVersion: "18528" uid: f525a217-35fc-47f9-9ac2-53ffc2a5f319 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:53:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18526" uid: 7323cfc1-e2fe-4a1a-a46f-43c84944ca00 kind: Event lastTimestamp: "2026-07-08T16:53:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:53:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:53:53Z" name: isvc-autogluon-v2-predictor.18c05eb619ebb336 namespace: kserve-ci-e2e-test resourceVersion: "18982" uid: 2ef18b6b-11e9-4163-81f9-7677f4fa701c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:53:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18526" uid: 7323cfc1-e2fe-4a1a-a46f-43c84944ca00 kind: Event lastTimestamp: "2026-07-08T16:53:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:53:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:53:53Z" name: isvc-autogluon-v2-predictor.18c05eb61a53d0d7 namespace: kserve-ci-e2e-test resourceVersion: "18983" uid: 1812c3b1-3911-4c7a-bb45-160ec35dacf6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:54:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18667" uid: 7323cfc1-e2fe-4a1a-a46f-43c84944ca00 kind: Event lastTimestamp: "2026-07-08T16:54:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:54:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:53Z" name: isvc-autogluon-v2-predictor.18c05ec4184dde61 namespace: kserve-ci-e2e-test resourceVersion: "19409" uid: afab9eba-2210-4d01-8bb7-bf306aa1cd59 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:54:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-predictor namespace: kserve-ci-e2e-test resourceVersion: "18667" uid: 7323cfc1-e2fe-4a1a-a46f-43c84944ca00 kind: Event lastTimestamp: "2026-07-08T16:54:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:54:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:53Z" name: isvc-autogluon-v2-predictor.18c05ec418b314b7 namespace: kserve-ci-e2e-test resourceVersion: "19410" uid: 26c40588-ff40-4681-acd9-5b7a58e15ee8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:54:59.060598Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5 namespace: kserve-ci-e2e-test resourceVersion: "19484" uid: 6bf36aa4-8d38-4847-ae5c-37a22b024fc7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5 to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-07-08T16:54:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:54:59Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5.18c05ecfceddebcb namespace: kserve-ci-e2e-test resourceVersion: "19489" uid: b6a0c6a6-840e-45e6-8be6-a3b2c447e6e9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-867dc68758-8x97b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5 namespace: kserve-ci-e2e-test resourceVersion: "19495" uid: 6bf36aa4-8d38-4847-ae5c-37a22b024fc7 kind: Event lastTimestamp: "2026-07-08T16:54:59Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:54:59Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5.18c05ecfe8bb00c4 namespace: kserve-ci-e2e-test resourceVersion: "19501" uid: ce39eb28-f5ac-4956-a232-e9861049945c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5 namespace: kserve-ci-e2e-test resourceVersion: "19486" uid: 6bf36aa4-8d38-4847-ae5c-37a22b024fc7 kind: Event lastTimestamp: "2026-07-08T16:54:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:43b18792ad3c8c70fe63e50dee47838dba7fd6b00a2dcfe15c76c1dafc2c7ad8" already present on machine metadata: creationTimestamp: "2026-07-08T16:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:59Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5.18c05ecfea3a53ef namespace: kserve-ci-e2e-test resourceVersion: "19503" uid: 8b8457f6-cfdc-4b74-bbec-b71abcda3e34 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5 namespace: kserve-ci-e2e-test resourceVersion: "19486" uid: 6bf36aa4-8d38-4847-ae5c-37a22b024fc7 kind: Event lastTimestamp: "2026-07-08T16:54:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:59Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5.18c05ecfed40fdf0 namespace: kserve-ci-e2e-test resourceVersion: "19505" uid: 1c6eeb95-ca49-490d-b873-17fcb7d4dbaf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5 namespace: kserve-ci-e2e-test resourceVersion: "19486" uid: 6bf36aa4-8d38-4847-ae5c-37a22b024fc7 kind: Event lastTimestamp: "2026-07-08T16:54:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:59Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5.18c05ecfedafa854 namespace: kserve-ci-e2e-test resourceVersion: "19506" uid: 6c4e3c35-447c-4dbb-81b6-76a21a1a7799 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5 namespace: kserve-ci-e2e-test resourceVersion: "19486" uid: 6bf36aa4-8d38-4847-ae5c-37a22b024fc7 kind: Event lastTimestamp: "2026-07-08T16:55:04Z" message: Container image "kserve/autogluonserver:latest" already present on machine metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:04Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5.18c05ed12c64fbea namespace: kserve-ci-e2e-test resourceVersion: "19549" uid: e6c5befb-2f36-413b-8d5b-963285ff4384 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5 namespace: kserve-ci-e2e-test resourceVersion: "19486" uid: 6bf36aa4-8d38-4847-ae5c-37a22b024fc7 kind: Event lastTimestamp: "2026-07-08T16:55:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:04Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5.18c05ed12f2a014b namespace: kserve-ci-e2e-test resourceVersion: "19551" uid: 301a11f6-4741-4f58-8428-f1d022e21ba2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5 namespace: kserve-ci-e2e-test resourceVersion: "19486" uid: 6bf36aa4-8d38-4847-ae5c-37a22b024fc7 kind: Event lastTimestamp: "2026-07-08T16:55:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:04Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5.18c05ed12f926499 namespace: kserve-ci-e2e-test resourceVersion: "19552" uid: b23b4646-1024-44e8-8155-1a358eaa2a84 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5 namespace: kserve-ci-e2e-test resourceVersion: "19486" uid: 6bf36aa4-8d38-4847-ae5c-37a22b024fc7 kind: Event lastTimestamp: "2026-07-08T16:55:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:55:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:04Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5.18c05ed12f9d9bcf namespace: kserve-ci-e2e-test resourceVersion: "19554" uid: 56ffa921-0bdf-489d-a277-78e4a013370b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5 namespace: kserve-ci-e2e-test resourceVersion: "19486" uid: 6bf36aa4-8d38-4847-ae5c-37a22b024fc7 kind: Event lastTimestamp: "2026-07-08T16:55:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:05Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5.18c05ed132e49021 namespace: kserve-ci-e2e-test resourceVersion: "19557" uid: 3e2961e9-8887-421c-8b9a-901f1463b985 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5 namespace: kserve-ci-e2e-test resourceVersion: "19486" uid: 6bf36aa4-8d38-4847-ae5c-37a22b024fc7 kind: Event lastTimestamp: "2026-07-08T16:55:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:05Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5.18c05ed13349ff07 namespace: kserve-ci-e2e-test resourceVersion: "19558" uid: 27147387-997c-4e9c-9e88-9379ea14fa32 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5 namespace: kserve-ci-e2e-test resourceVersion: "19486" uid: 6bf36aa4-8d38-4847-ae5c-37a22b024fc7 kind: Event lastTimestamp: "2026-07-08T16:56:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:56:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:56:39Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5.18c05ee71ed16ed9 namespace: kserve-ci-e2e-test resourceVersion: "20241" uid: d1b210f8-e645-4e97-8125-11b4b8492238 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5 namespace: kserve-ci-e2e-test resourceVersion: "19486" uid: 6bf36aa4-8d38-4847-ae5c-37a22b024fc7 kind: Event lastTimestamp: "2026-07-08T16:56:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:56:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:56:39Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5.18c05ee71ed22ad2 namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: 0b0fd94a-af03-4abd-a15c-7538d91780b8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-autogluon-v2-variants-predictor-c6767c94c namespace: kserve-ci-e2e-test resourceVersion: "19475" uid: 479a441a-45cf-4d57-ab38-49abd7c3df43 kind: Event lastTimestamp: "2026-07-08T16:54:59Z" message: 'Created pod: isvc-autogluon-v2-variants-predictor-c6767c94c-wz2v5' metadata: creationTimestamp: "2026-07-08T16:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:59Z" name: isvc-autogluon-v2-variants-predictor-c6767c94c.18c05ecfce3badd2 namespace: kserve-ci-e2e-test resourceVersion: "19485" uid: e9a0dc36-5419-4d62-9822-370930ce2b52 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "19473" uid: 50364162-3876-40ac-b771-29692b2d627e kind: Event lastTimestamp: "2026-07-08T16:54:58Z" message: Scaled up replica set isvc-autogluon-v2-variants-predictor-c6767c94c from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:58Z" name: isvc-autogluon-v2-variants-predictor.18c05ecfc9b99ea5 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: 6e50c952-37b2-4fbb-80f6-185018196fd0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:55:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "19474" uid: 00f06c30-bf9a-4dc8-a236-e79fd81e0d37 kind: Event lastTimestamp: "2026-07-08T16:55:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:29Z" name: isvc-autogluon-v2-variants-predictor.18c05ed348d6d0f4 namespace: kserve-ci-e2e-test resourceVersion: "19726" uid: 0d760265-8664-4d8e-a0eb-183c22cb0a97 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:55:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "19474" uid: 00f06c30-bf9a-4dc8-a236-e79fd81e0d37 kind: Event lastTimestamp: "2026-07-08T16:55:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:29Z" name: isvc-autogluon-v2-variants-predictor.18c05ed3494ad1d7 namespace: kserve-ci-e2e-test resourceVersion: "19727" uid: 56bda6b7-6ce9-463e-af41-39f85056f2e6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "19624" uid: 00f06c30-bf9a-4dc8-a236-e79fd81e0d37 kind: Event lastTimestamp: "2026-07-08T16:56:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:29Z" name: isvc-autogluon-v2-variants-predictor.18c05eda4787d6b0 namespace: kserve-ci-e2e-test resourceVersion: "20142" uid: d79c72ec-43e7-4c8d-ac7c-81a7ad6c75ce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:55:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-autogluon-v2-variants-predictor namespace: kserve-ci-e2e-test resourceVersion: "19624" uid: 00f06c30-bf9a-4dc8-a236-e79fd81e0d37 kind: Event lastTimestamp: "2026-07-08T16:56:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:29Z" name: isvc-autogluon-v2-variants-predictor.18c05eda47fbcbc5 namespace: kserve-ci-e2e-test resourceVersion: "20144" uid: cbbf689c-e21b-4c83-8fd6-6029c479997c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-08T16:54:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: 0d949868-b43b-4e30-8a93-0779f74ddfb4 kind: Event lastTimestamp: "2026-07-08T16:56:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:56:37Z" name: isvc-autogluon-v2-variants.18c05ecfc9e1e1fb namespace: kserve-ci-e2e-test resourceVersion: "20211" uid: d91ee2e3-1585-4bae-9cfc-3cdb4f39950c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: 0d949868-b43b-4e30-8a93-0779f74ddfb4 kind: Event lastTimestamp: "2026-07-08T16:54:58Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2-variants": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:54:58Z" name: isvc-autogluon-v2-variants.18c05ecfca3cee8e namespace: kserve-ci-e2e-test resourceVersion: "19479" uid: 80d74088-4ef4-48c9-881f-81d13729cc4a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: 0d949868-b43b-4e30-8a93-0779f74ddfb4 kind: Event lastTimestamp: "2026-07-08T16:54:58Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2-variants": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:54:58Z" name: isvc-autogluon-v2-variants.18c05ecfca3d480c namespace: kserve-ci-e2e-test resourceVersion: "19480" uid: 5f8183c4-207b-45b2-abf4-deaef006086c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2-variants namespace: kserve-ci-e2e-test resourceVersion: "20209" uid: 0d949868-b43b-4e30-8a93-0779f74ddfb4 kind: Event lastTimestamp: "2026-07-08T16:56:37Z" message: InferenceService [isvc-autogluon-v2-variants] is Ready metadata: creationTimestamp: "2026-07-08T16:56:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:56:37Z" name: isvc-autogluon-v2-variants.18c05ee6ae091330 namespace: kserve-ci-e2e-test resourceVersion: "20210" uid: 4188332b-609a-44a0-a362-2fa5ebd7c9ce reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-08T16:52:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "18517" uid: 26f49af1-e461-443c-b0d3-41fc8c5465a2 kind: Event lastTimestamp: "2026-07-08T16:54:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:52:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:54:53Z" name: isvc-autogluon-v2.18c05eb29af3b0c1 namespace: kserve-ci-e2e-test resourceVersion: "19402" uid: 1fcaef01-0505-420c-9811-887f989b5e61 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:52:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "18517" uid: 26f49af1-e461-443c-b0d3-41fc8c5465a2 kind: Event lastTimestamp: "2026-07-08T16:52:53Z" message: 'Failed to update status for InferenceService "isvc-autogluon-v2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:52:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:52:53Z" name: isvc-autogluon-v2.18c05eb29b95f1bc namespace: kserve-ci-e2e-test resourceVersion: "18548" uid: 83d920e1-93f0-4b7b-b165-77fa3cea427c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:52:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "18517" uid: 26f49af1-e461-443c-b0d3-41fc8c5465a2 kind: Event lastTimestamp: "2026-07-08T16:52:53Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-autogluon-v2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:52:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:52:53Z" name: isvc-autogluon-v2.18c05eb29b96345b namespace: kserve-ci-e2e-test resourceVersion: "18549" uid: 13eb3ca0-f07f-40e9-8051-411fab1130e5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-autogluon-v2 namespace: kserve-ci-e2e-test resourceVersion: "19400" uid: 26f49af1-e461-443c-b0d3-41fc8c5465a2 kind: Event lastTimestamp: "2026-07-08T16:54:53Z" message: InferenceService [isvc-autogluon-v2] is Ready metadata: creationTimestamp: "2026-07-08T16:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:54:53Z" name: isvc-autogluon-v2.18c05ece78a1080f namespace: kserve-ci-e2e-test resourceVersion: "19401" uid: e6ca5d9f-50c0-457b-9892-e5a677594451 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:48:32.783390Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-786d9c95cd-vznlq namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 06bfd315-7e36-4511-b9d7-c2f488b06b8a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-predictor-786d9c95cd-vznlq to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-07-08T16:48:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:48:32Z" name: isvc-logger-predictor-786d9c95cd-vznlq.18c05e75def36f65 namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: 5aa3e6a1-4eec-45c9-997d-edeedb72c3a5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-867dc68758-8x97b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:33Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-predictor-786d9c95cd-vznlq namespace: kserve-ci-e2e-test resourceVersion: "16496" uid: 06bfd315-7e36-4511-b9d7-c2f488b06b8a kind: Event lastTimestamp: "2026-07-08T16:48:33Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:48:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:48:33Z" name: isvc-logger-predictor-786d9c95cd-vznlq.18c05e75f92192bd namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 4f3b8139-adfb-4465-980d-55f2a551c5d0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-786d9c95cd-vznlq namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 06bfd315-7e36-4511-b9d7-c2f488b06b8a kind: Event lastTimestamp: "2026-07-08T16:48:33Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:43b18792ad3c8c70fe63e50dee47838dba7fd6b00a2dcfe15c76c1dafc2c7ad8" already present on machine metadata: creationTimestamp: "2026-07-08T16:48:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:33Z" name: isvc-logger-predictor-786d9c95cd-vznlq.18c05e75fabd3b75 namespace: kserve-ci-e2e-test resourceVersion: "16507" uid: 87de1648-f828-4c43-bd56-0e6c9763ac70 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-786d9c95cd-vznlq namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 06bfd315-7e36-4511-b9d7-c2f488b06b8a kind: Event lastTimestamp: "2026-07-08T16:48:33Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:48:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:33Z" name: isvc-logger-predictor-786d9c95cd-vznlq.18c05e75fda50725 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 86e3adb6-d591-40f1-980d-86a7cc959235 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-predictor-786d9c95cd-vznlq namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 06bfd315-7e36-4511-b9d7-c2f488b06b8a kind: Event lastTimestamp: "2026-07-08T16:48:33Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:48:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:33Z" name: isvc-logger-predictor-786d9c95cd-vznlq.18c05e75fe1da9e5 namespace: kserve-ci-e2e-test resourceVersion: "16510" uid: e924ca7f-caa0-45c9-8c76-37776b3e032e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-786d9c95cd-vznlq namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 06bfd315-7e36-4511-b9d7-c2f488b06b8a kind: Event lastTimestamp: "2026-07-08T16:48:37Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1707" already present on machine metadata: creationTimestamp: "2026-07-08T16:48:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:37Z" name: isvc-logger-predictor-786d9c95cd-vznlq.18c05e76dd770944 namespace: kserve-ci-e2e-test resourceVersion: "16540" uid: 6790b3cc-592c-47cf-b80a-7cae6a82d250 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-786d9c95cd-vznlq namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 06bfd315-7e36-4511-b9d7-c2f488b06b8a kind: Event lastTimestamp: "2026-07-08T16:48:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:48:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:37Z" name: isvc-logger-predictor-786d9c95cd-vznlq.18c05e76e0848312 namespace: kserve-ci-e2e-test resourceVersion: "16544" uid: 6479c0b8-262a-4ce3-9bdf-0cef443316b9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-786d9c95cd-vznlq namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 06bfd315-7e36-4511-b9d7-c2f488b06b8a kind: Event lastTimestamp: "2026-07-08T16:48:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:48:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:37Z" name: isvc-logger-predictor-786d9c95cd-vznlq.18c05e76e0fe5156 namespace: kserve-ci-e2e-test resourceVersion: "16545" uid: 9cb54534-9757-4791-a843-e6bb28f1af1a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-786d9c95cd-vznlq namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 06bfd315-7e36-4511-b9d7-c2f488b06b8a kind: Event lastTimestamp: "2026-07-08T16:48:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:48:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:37Z" name: isvc-logger-predictor-786d9c95cd-vznlq.18c05e76e107d501 namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 83c28ee1-9fc1-4862-9520-be087c491063 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-786d9c95cd-vznlq namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 06bfd315-7e36-4511-b9d7-c2f488b06b8a kind: Event lastTimestamp: "2026-07-08T16:48:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:48:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:37Z" name: isvc-logger-predictor-786d9c95cd-vznlq.18c05e76e4047696 namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: 4f7c767f-c8ab-4fd6-948e-be605aec06a5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-786d9c95cd-vznlq namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 06bfd315-7e36-4511-b9d7-c2f488b06b8a kind: Event lastTimestamp: "2026-07-08T16:48:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:48:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:37Z" name: isvc-logger-predictor-786d9c95cd-vznlq.18c05e76e480e0f6 namespace: kserve-ci-e2e-test resourceVersion: "16551" uid: f72f43fc-725e-48fa-9f6f-703632aa7f06 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-786d9c95cd-vznlq namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 06bfd315-7e36-4511-b9d7-c2f488b06b8a kind: Event lastTimestamp: "2026-07-08T16:48:37Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:7cf57df6438fd0be0182caf8c665267f434f07d282f75228157fde800aa58082" already present on machine metadata: creationTimestamp: "2026-07-08T16:48:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:37Z" name: isvc-logger-predictor-786d9c95cd-vznlq.18c05e76e48b4c96 namespace: kserve-ci-e2e-test resourceVersion: "16552" uid: 6fd0b107-91cd-453b-8dc6-2c7383112bdc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-786d9c95cd-vznlq namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 06bfd315-7e36-4511-b9d7-c2f488b06b8a kind: Event lastTimestamp: "2026-07-08T16:48:37Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-08T16:48:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:37Z" name: isvc-logger-predictor-786d9c95cd-vznlq.18c05e76e7c450fd namespace: kserve-ci-e2e-test resourceVersion: "16553" uid: 7011d10e-6965-4323-8ef5-59f11fff6b26 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-786d9c95cd-vznlq namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 06bfd315-7e36-4511-b9d7-c2f488b06b8a kind: Event lastTimestamp: "2026-07-08T16:48:37Z" message: Started container agent metadata: creationTimestamp: "2026-07-08T16:48:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:37Z" name: isvc-logger-predictor-786d9c95cd-vznlq.18c05e76e83d4fab namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: a845b302-7419-4949-a8a3-8811455c03fd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-08T16:48:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-786d9c95cd-vznlq namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 06bfd315-7e36-4511-b9d7-c2f488b06b8a kind: Event lastTimestamp: "2026-07-08T16:50:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:48:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:24Z" name: isvc-logger-predictor-786d9c95cd-vznlq.18c05e77198feabf namespace: kserve-ci-e2e-test resourceVersion: "17431" uid: 591074cf-141b-4691-bada-e11393939d42 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T16:48:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-786d9c95cd-vznlq namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 06bfd315-7e36-4511-b9d7-c2f488b06b8a kind: Event lastTimestamp: "2026-07-08T16:50:14Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T16:48:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:14Z" name: isvc-logger-predictor-786d9c95cd-vznlq.18c05e7719992383 namespace: kserve-ci-e2e-test resourceVersion: "17358" uid: 5589af21-c7f5-419b-863e-3fb8d5861a85 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-predictor-786d9c95cd-vznlq namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 06bfd315-7e36-4511-b9d7-c2f488b06b8a kind: Event lastTimestamp: "2026-07-08T16:50:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:08Z" name: isvc-logger-predictor-786d9c95cd-vznlq.18c05e8c0ee611d8 namespace: kserve-ci-e2e-test resourceVersion: "17258" uid: 3413d332-9e00-4f08-a187-cb615e02170c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-predictor-786d9c95cd-vznlq namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 06bfd315-7e36-4511-b9d7-c2f488b06b8a kind: Event lastTimestamp: "2026-07-08T16:50:08Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-08T16:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:08Z" name: isvc-logger-predictor-786d9c95cd-vznlq.18c05e8c0ee641a8 namespace: kserve-ci-e2e-test resourceVersion: "17261" uid: c11ac482-6725-4ac2-a49c-5bc160d8eba0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-786d9c95cd-vznlq namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 06bfd315-7e36-4511-b9d7-c2f488b06b8a kind: Event lastTimestamp: "2026-07-08T16:50:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:08Z" name: isvc-logger-predictor-786d9c95cd-vznlq.18c05e8c0ee675f2 namespace: kserve-ci-e2e-test resourceVersion: "17262" uid: a186d1ae-804f-4c4e-a8b8-ee7994af69c7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:50:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-predictor-786d9c95cd-vznlq namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: 06bfd315-7e36-4511-b9d7-c2f488b06b8a kind: Event lastTimestamp: "2026-07-08T16:50:24Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:50:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:24Z" name: isvc-logger-predictor-786d9c95cd-vznlq.18c05e8c49d44e75 namespace: kserve-ci-e2e-test resourceVersion: "17430" uid: b9b74f0f-5cce-4206-aece-aea63d294131 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-predictor-786d9c95cd namespace: kserve-ci-e2e-test resourceVersion: "16475" uid: 86e92e03-2486-45ea-b001-fab7ceb91e74 kind: Event lastTimestamp: "2026-07-08T16:48:32Z" message: 'Created pod: isvc-logger-predictor-786d9c95cd-vznlq' metadata: creationTimestamp: "2026-07-08T16:48:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:32Z" name: isvc-logger-predictor-786d9c95cd.18c05e75de754978 namespace: kserve-ci-e2e-test resourceVersion: "16488" uid: a7306a75-16fa-43db-aafb-a3aef5fd27c5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16473" uid: 8d24c7bd-2b3a-4aa5-9962-ba33a535aad4 kind: Event lastTimestamp: "2026-07-08T16:48:32Z" message: Scaled up replica set isvc-logger-predictor-786d9c95cd from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:48:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:32Z" name: isvc-logger-predictor.18c05e75da91fbd8 namespace: kserve-ci-e2e-test resourceVersion: "16477" uid: 3c01817f-4e71-4209-a46f-b07b3466e1ba reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:48:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16474" uid: ab0549a8-d160-4bf3-8632-1ba660e2eb38 kind: Event lastTimestamp: "2026-07-08T16:49:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:49:02Z" name: isvc-logger-predictor.18c05e79592da2ba namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 5db5e16e-f05d-4315-9510-4b27aeae8b47 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:48:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16474" uid: ab0549a8-d160-4bf3-8632-1ba660e2eb38 kind: Event lastTimestamp: "2026-07-08T16:49:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:49:02Z" name: isvc-logger-predictor.18c05e79599d35f7 namespace: kserve-ci-e2e-test resourceVersion: "16749" uid: f9f09ba9-edfd-4c54-ae65-de57cb606a31 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:49:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16641" uid: ab0549a8-d160-4bf3-8632-1ba660e2eb38 kind: Event lastTimestamp: "2026-07-08T16:50:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:49:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:50:02Z" name: isvc-logger-predictor.18c05e80575079dc namespace: kserve-ci-e2e-test resourceVersion: "17182" uid: f330d25b-9327-4bcd-9425-3342b67c495c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:49:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-predictor namespace: kserve-ci-e2e-test resourceVersion: "16641" uid: ab0549a8-d160-4bf3-8632-1ba660e2eb38 kind: Event lastTimestamp: "2026-07-08T16:50:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:49:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:50:02Z" name: isvc-logger-predictor.18c05e8057c0dfe4 namespace: kserve-ci-e2e-test resourceVersion: "17183" uid: 1b11f701-4894-4fa0-b076-cd30b7bd181b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-08T16:48:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "16464" uid: d9dc6d16-cc03-4ec6-ac9d-f79fdb6c3528 kind: Event lastTimestamp: "2026-07-08T16:49:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:48:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:49:54Z" name: isvc-logger.18c05e75da93f5af namespace: kserve-ci-e2e-test resourceVersion: "17123" uid: 127ed496-6fe0-4d8c-a224-0a59ca8ad870 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "16464" uid: d9dc6d16-cc03-4ec6-ac9d-f79fdb6c3528 kind: Event lastTimestamp: "2026-07-08T16:48:32Z" message: 'Failed to update status for InferenceService "isvc-logger": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:48:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:48:32Z" name: isvc-logger.18c05e75db0fa5c9 namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 681d7f40-b0a3-4529-9614-40e4a4195c07 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger namespace: kserve-ci-e2e-test resourceVersion: "17121" uid: d9dc6d16-cc03-4ec6-ac9d-f79fdb6c3528 kind: Event lastTimestamp: "2026-07-08T16:49:54Z" message: InferenceService [isvc-logger] is Ready metadata: creationTimestamp: "2026-07-08T16:49:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:49:54Z" name: isvc-logger.18c05e88d2cc5294 namespace: kserve-ci-e2e-test resourceVersion: "17122" uid: c7929416-ad8e-41ad-8943-766a891ab150 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:46:37.548691Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 namespace: kserve-ci-e2e-test resourceVersion: "15488" uid: 7cf5ee2c-2516-4231-af20-dd1c8b77aaa6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-07-08T16:46:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:46:37Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6.18c05e5b0a6d8b1f namespace: kserve-ci-e2e-test resourceVersion: "15492" uid: a726eb0b-eddf-41c0-98e5-f2d567bc04e4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-867dc68758-8x97b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:37Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 namespace: kserve-ci-e2e-test resourceVersion: "15499" uid: 7cf5ee2c-2516-4231-af20-dd1c8b77aaa6 kind: Event lastTimestamp: "2026-07-08T16:46:37Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:46:37Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6.18c05e5b246fcd17 namespace: kserve-ci-e2e-test resourceVersion: "15506" uid: 12f8e9e9-49f6-4e47-a8c4-b4234f306c5e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 7cf5ee2c-2516-4231-af20-dd1c8b77aaa6 kind: Event lastTimestamp: "2026-07-08T16:46:38Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:43b18792ad3c8c70fe63e50dee47838dba7fd6b00a2dcfe15c76c1dafc2c7ad8" already present on machine metadata: creationTimestamp: "2026-07-08T16:46:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:38Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6.18c05e5b2606dcf9 namespace: kserve-ci-e2e-test resourceVersion: "15508" uid: 4c567245-1b89-4d17-8fd1-2bb72c9a47ce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 7cf5ee2c-2516-4231-af20-dd1c8b77aaa6 kind: Event lastTimestamp: "2026-07-08T16:46:38Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:46:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:38Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6.18c05e5b291d5e96 namespace: kserve-ci-e2e-test resourceVersion: "15509" uid: 498f6f1d-3f0c-4ead-82b4-9670ea8df22f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:38Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 7cf5ee2c-2516-4231-af20-dd1c8b77aaa6 kind: Event lastTimestamp: "2026-07-08T16:46:38Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:46:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:38Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6.18c05e5b29ba65c4 namespace: kserve-ci-e2e-test resourceVersion: "15510" uid: f168cd6b-ab31-41a7-b975-142e79af8166 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 7cf5ee2c-2516-4231-af20-dd1c8b77aaa6 kind: Event lastTimestamp: "2026-07-08T16:46:42Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1707" already present on machine metadata: creationTimestamp: "2026-07-08T16:46:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:42Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6.18c05e5c330da00b namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: 4435aacd-edfe-4b93-977e-b62979b3dad2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 7cf5ee2c-2516-4231-af20-dd1c8b77aaa6 kind: Event lastTimestamp: "2026-07-08T16:46:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:46:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:42Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6.18c05e5c35f30c5a namespace: kserve-ci-e2e-test resourceVersion: "15558" uid: e031c5c0-6a1b-48e1-a3ff-396b9b808850 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 7cf5ee2c-2516-4231-af20-dd1c8b77aaa6 kind: Event lastTimestamp: "2026-07-08T16:46:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:46:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:42Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6.18c05e5c3684fa3d namespace: kserve-ci-e2e-test resourceVersion: "15561" uid: 43f82f7d-bd1e-4a77-a631-ef619597fe22 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 7cf5ee2c-2516-4231-af20-dd1c8b77aaa6 kind: Event lastTimestamp: "2026-07-08T16:46:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:46:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:42Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6.18c05e5c3690403e namespace: kserve-ci-e2e-test resourceVersion: "15562" uid: 9eab751b-7227-40d9-8d7d-02a74f015ed7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 7cf5ee2c-2516-4231-af20-dd1c8b77aaa6 kind: Event lastTimestamp: "2026-07-08T16:46:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:46:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:42Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6.18c05e5c3932fb7a namespace: kserve-ci-e2e-test resourceVersion: "15564" uid: 84e77f2b-e5dd-4d5e-b7d3-92241c00a486 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 7cf5ee2c-2516-4231-af20-dd1c8b77aaa6 kind: Event lastTimestamp: "2026-07-08T16:46:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:46:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:42Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6.18c05e5c3997e98a namespace: kserve-ci-e2e-test resourceVersion: "15565" uid: ef33ae4e-f912-4df9-9a50-5da2c6c86749 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 7cf5ee2c-2516-4231-af20-dd1c8b77aaa6 kind: Event lastTimestamp: "2026-07-08T16:46:42Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:7cf57df6438fd0be0182caf8c665267f434f07d282f75228157fde800aa58082" already present on machine metadata: creationTimestamp: "2026-07-08T16:46:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:42Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6.18c05e5c39a374a0 namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: 11fb5ee3-59cb-4c1b-94a7-383d15cbac41 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 7cf5ee2c-2516-4231-af20-dd1c8b77aaa6 kind: Event lastTimestamp: "2026-07-08T16:46:42Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-08T16:46:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:42Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6.18c05e5c3ccb6132 namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: dcea1ad4-39f8-42db-b7ff-4522fb4fee8e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 7cf5ee2c-2516-4231-af20-dd1c8b77aaa6 kind: Event lastTimestamp: "2026-07-08T16:46:42Z" message: Started container agent metadata: creationTimestamp: "2026-07-08T16:46:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:42Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6.18c05e5c3d3904b2 namespace: kserve-ci-e2e-test resourceVersion: "15569" uid: 6fdd7195-019d-430d-bf25-7ad5e3cdb9f5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-08T16:46:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 7cf5ee2c-2516-4231-af20-dd1c8b77aaa6 kind: Event lastTimestamp: "2026-07-08T16:48:20Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:5000: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:46:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:20Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6.18c05e5c6f21ec53 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 6120bacf-558e-40d2-8d7a-e754e3ba584b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T16:46:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 7cf5ee2c-2516-4231-af20-dd1c8b77aaa6 kind: Event lastTimestamp: "2026-07-08T16:48:20Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T16:46:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:20Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6.18c05e5cab04ef03 namespace: kserve-ci-e2e-test resourceVersion: "16365" uid: 670412e1-c907-402b-a6e8-e4794733f8f0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 7cf5ee2c-2516-4231-af20-dd1c8b77aaa6 kind: Event lastTimestamp: "2026-07-08T16:48:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:12Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6.18c05e712faabddd namespace: kserve-ci-e2e-test resourceVersion: "16257" uid: b8b1e7fb-1e50-40ef-8c32-82826cf30f0e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 7cf5ee2c-2516-4231-af20-dd1c8b77aaa6 kind: Event lastTimestamp: "2026-07-08T16:48:12Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-08T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:12Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6.18c05e712fabe2d6 namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: 2b24377f-3e7e-4ddf-bb35-327430e60f3d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 7cf5ee2c-2516-4231-af20-dd1c8b77aaa6 kind: Event lastTimestamp: "2026-07-08T16:48:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:12Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6.18c05e712fac2f93 namespace: kserve-ci-e2e-test resourceVersion: "16261" uid: 6db0aabb-dd6c-45ce-b6ed-9270ae9e23de reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6 namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 7cf5ee2c-2516-4231-af20-dd1c8b77aaa6 kind: Event lastTimestamp: "2026-07-08T16:48:30Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:30Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6.18c05e71db43ae28 namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: 6042e496-ef05-40d3-a32d-8d1f28c19ac1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-custom-predictor-86d75f96d4 namespace: kserve-ci-e2e-test resourceVersion: "15471" uid: f2084964-0f0b-4e37-b28b-5c48c23ad00a kind: Event lastTimestamp: "2026-07-08T16:46:37Z" message: 'Created pod: isvc-sklearn-batcher-custom-predictor-86d75f96d4-zcrk6' metadata: creationTimestamp: "2026-07-08T16:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:37Z" name: isvc-sklearn-batcher-custom-predictor-86d75f96d4.18c05e5b09cbaf4c namespace: kserve-ci-e2e-test resourceVersion: "15491" uid: b0cf4925-7fab-4ac4-8705-8a1d148e33e1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15468" uid: 8382158f-9726-41ef-9032-88fa5e20a1bf kind: Event lastTimestamp: "2026-07-08T16:46:37Z" message: Scaled up replica set isvc-sklearn-batcher-custom-predictor-86d75f96d4 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:37Z" name: isvc-sklearn-batcher-custom-predictor.18c05e5b03a6c327 namespace: kserve-ci-e2e-test resourceVersion: "15477" uid: ec4ec0b3-d65a-4143-ba2f-fc1b36a9385a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:46:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15469" uid: 72b590f5-b38f-4169-8ed3-70779b3aab1b kind: Event lastTimestamp: "2026-07-08T16:47:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:47:22Z" name: isvc-sklearn-batcher-custom-predictor.18c05e5e820d5bb5 namespace: kserve-ci-e2e-test resourceVersion: "15864" uid: 578fd441-ee7a-4d82-8e28-38df3764d9b5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:46:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15469" uid: 72b590f5-b38f-4169-8ed3-70779b3aab1b kind: Event lastTimestamp: "2026-07-08T16:47:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:46:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:47:22Z" name: isvc-sklearn-batcher-custom-predictor.18c05e5e82806785 namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: 65d8b4da-aecb-4743-b661-fc1642f41bc3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:47:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 72b590f5-b38f-4169-8ed3-70779b3aab1b kind: Event lastTimestamp: "2026-07-08T16:48:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:47:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:07Z" name: isvc-sklearn-batcher-custom-predictor.18c05e620272f06b namespace: kserve-ci-e2e-test resourceVersion: "16196" uid: fd0df80a-346b-42b9-bbde-9453f3da0e61 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:47:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-custom-predictor namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 72b590f5-b38f-4169-8ed3-70779b3aab1b kind: Event lastTimestamp: "2026-07-08T16:48:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:47:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:07Z" name: isvc-sklearn-batcher-custom-predictor.18c05e6202f6bc81 namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: 6d2f4b05-98ce-495e-8192-f40cfd2d8d58 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-08T16:46:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "15445" uid: 5097646b-af88-4539-8f8d-65a7ad43f018 kind: Event lastTimestamp: "2026-07-08T16:48:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:48:00Z" name: isvc-sklearn-batcher-custom.18c05e5b039fd21d namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: 0df042fc-dcd6-4dad-8e00-9e80a6cbbb74 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:46:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "15445" uid: 5097646b-af88-4539-8f8d-65a7ad43f018 kind: Event lastTimestamp: "2026-07-08T16:46:37Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher-custom": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher-custom": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:46:37Z" name: isvc-sklearn-batcher-custom.18c05e5b0436cc1f namespace: kserve-ci-e2e-test resourceVersion: "15497" uid: 92fd50c4-809e-4c17-9c45-6f2994856182 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "15451" uid: 5097646b-af88-4539-8f8d-65a7ad43f018 kind: Event lastTimestamp: "2026-07-08T16:46:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher-custom": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:46:37Z" name: isvc-sklearn-batcher-custom.18c05e5b0b65c7ea namespace: kserve-ci-e2e-test resourceVersion: "15498" uid: d1096a6c-4fe1-40ef-80d2-12ab23c35d78 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher-custom namespace: kserve-ci-e2e-test resourceVersion: "16149" uid: 5097646b-af88-4539-8f8d-65a7ad43f018 kind: Event lastTimestamp: "2026-07-08T16:48:00Z" message: InferenceService [isvc-sklearn-batcher-custom] is Ready metadata: creationTimestamp: "2026-07-08T16:48:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:48:00Z" name: isvc-sklearn-batcher-custom.18c05e6e65af8f41 namespace: kserve-ci-e2e-test resourceVersion: "16150" uid: dd0b054c-8b85-413c-a5d1-653c55ee557e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:44:52.384821Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-07-08T16:44:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:44:52Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e428e2c3494 namespace: kserve-ci-e2e-test resourceVersion: "14654" uid: 8db033d6-e8e2-4388-bd87-8d3bb6d9da32 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-867dc68758-8x97b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14661" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:44:52Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:44:52Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e42a7e3ccb0 namespace: kserve-ci-e2e-test resourceVersion: "14670" uid: 587b9f0e-69f6-4dee-a65b-7b02da30fce4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:44:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:43b18792ad3c8c70fe63e50dee47838dba7fd6b00a2dcfe15c76c1dafc2c7ad8" already present on machine metadata: creationTimestamp: "2026-07-08T16:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:52Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e42a9d3d9e1 namespace: kserve-ci-e2e-test resourceVersion: "14672" uid: b869a0bf-40c8-4930-8107-286fa2935191 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:44:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:52Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e42acfa241b namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: 7dd98e11-c04f-47b5-9aa2-b861d8879723 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:44:52Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:52Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e42ad6620f7 namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: c70ab56a-5b75-4072-ab8a-8f2890f95842 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:44:57Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1707" metadata: creationTimestamp: "2026-07-08T16:44:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:44:57Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e43a2c96b88 namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: 9ba0cace-e34a-4de4-a8cb-9b8c99794f9e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:45:11Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1707" in 14.491s (14.491s including waiting). Image size: 1555648957 bytes.' metadata: creationTimestamp: "2026-07-08T16:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:11Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e47028fc588 namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: 1f0383ea-358b-4963-a825-0e794b8ea9b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:45:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:11Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e4705c41c2d namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: 531933b5-a4da-40b1-abbb-8e79e0a21fc9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:45:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:11Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e47062e50ee namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: 7baf59fa-d544-45ef-a2e0-38478ad6d2eb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:45:11Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-08T16:45:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:11Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e47063d3726 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: 64f6e2ee-d255-4103-9391-6c2075da1f9b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:45:14Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.533s (2.533s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-08T16:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:14Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e479d43d733 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: 15844243-3f67-4b60-8891-e3e88c7962cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:45:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:14Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e47a02d36dd namespace: kserve-ci-e2e-test resourceVersion: "14835" uid: 5cae6e5b-d210-42a6-a690-198a59353824 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:45:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:14Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e47a094891a namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: c7f22ff4-9564-473b-84f7-23b94c53eaa1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:45:14Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:7cf57df6438fd0be0182caf8c665267f434f07d282f75228157fde800aa58082" metadata: creationTimestamp: "2026-07-08T16:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:14Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e47a0a232ab namespace: kserve-ci-e2e-test resourceVersion: "14837" uid: e55e3338-384a-4066-bca9-1e0a4a94d624 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:45:17Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:7cf57df6438fd0be0182caf8c665267f434f07d282f75228157fde800aa58082" in 3.319s (3.319s including waiting). Image size: 239997061 bytes.' metadata: creationTimestamp: "2026-07-08T16:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:17Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e48667e2ec2 namespace: kserve-ci-e2e-test resourceVersion: "14858" uid: 82560411-6b74-4d65-b558-b55932024ccc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:45:17Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-08T16:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:17Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e4869508be3 namespace: kserve-ci-e2e-test resourceVersion: "14860" uid: 05f5b4b4-4b18-448c-9829-c640b9a49014 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:45:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:45:17Z" message: Started container agent metadata: creationTimestamp: "2026-07-08T16:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:45:17Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e4869b783f9 namespace: kserve-ci-e2e-test resourceVersion: "14861" uid: 4aee7ebc-adbc-461a-93da-d052d8f83731 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:46:51Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:51Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e48c956dbb4 namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: 2c22a3a4-9c46-488d-85b5-cc479fa323c5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-08T16:45:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:46:51Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T16:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:51Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e48c9635f0d namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: 02ef35e5-5ac6-4991-9647-5773379d180f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:46:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:37Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e5b02e6cd06 namespace: kserve-ci-e2e-test resourceVersion: "15470" uid: fa21d68f-a6b6-4569-b33d-d6c7329ae89a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:46:37Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-08T16:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:37Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e5b02e709c8 namespace: kserve-ci-e2e-test resourceVersion: "15472" uid: 51375dde-2d42-4ba0-8947-bcb8c3558038 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:46:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:37Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e5b02e77edb namespace: kserve-ci-e2e-test resourceVersion: "15475" uid: 29d91932-0bae-44dd-ba5e-a2bac84f8d2e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:46:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 01515c8e-ff97-4e25-87ad-05dbd49cb8d2 kind: Event lastTimestamp: "2026-07-08T16:46:49Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:46:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:46:49Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d.18c05e5b69a9317a namespace: kserve-ci-e2e-test resourceVersion: "15613" uid: fa702aeb-ff24-45c4-9016-2565413f2b0b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-batcher-predictor-7bcdbf6fbf namespace: kserve-ci-e2e-test resourceVersion: "14638" uid: c70bd31d-baad-4160-bfe2-a2d2e6785422 kind: Event lastTimestamp: "2026-07-08T16:44:52Z" message: 'Created pod: isvc-sklearn-batcher-predictor-7bcdbf6fbf-llv6d' metadata: creationTimestamp: "2026-07-08T16:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:44:52Z" name: isvc-sklearn-batcher-predictor-7bcdbf6fbf.18c05e428d82d98c namespace: kserve-ci-e2e-test resourceVersion: "14651" uid: 5a6a2152-4226-42af-a573-f2092dcefade reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14637" uid: 63fbe94e-e8e7-4583-b8cc-518cde587fa0 kind: Event lastTimestamp: "2026-07-08T16:44:52Z" message: Scaled up replica set isvc-sklearn-batcher-predictor-7bcdbf6fbf from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:44:52Z" name: isvc-sklearn-batcher-predictor.18c05e4284ebc090 namespace: kserve-ci-e2e-test resourceVersion: "14639" uid: d807f5f5-9d79-4d74-8668-c39150d3a3af reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:45:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14645" uid: be1111a7-c7b3-4993-bd58-32e344881663 kind: Event lastTimestamp: "2026-07-08T16:45:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:45:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:52Z" name: isvc-sklearn-batcher-predictor.18c05e460ab98d7d namespace: kserve-ci-e2e-test resourceVersion: "15122" uid: cc906f38-958a-4333-a4af-e043940d1bf8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:45:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14645" uid: be1111a7-c7b3-4993-bd58-32e344881663 kind: Event lastTimestamp: "2026-07-08T16:45:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:45:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:45:52Z" name: isvc-sklearn-batcher-predictor.18c05e460b0eaae5 namespace: kserve-ci-e2e-test resourceVersion: "15124" uid: 7da3c824-a95a-4810-8de4-28701edaa857 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:45:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14777" uid: be1111a7-c7b3-4993-bd58-32e344881663 kind: Event lastTimestamp: "2026-07-08T16:46:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:22Z" name: isvc-sklearn-batcher-predictor.18c05e4d09749215 namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: 0763fa6c-6485-4f0c-b6fb-c10cb4f29d28 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:45:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-batcher-predictor namespace: kserve-ci-e2e-test resourceVersion: "14777" uid: be1111a7-c7b3-4993-bd58-32e344881663 kind: Event lastTimestamp: "2026-07-08T16:46:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:46:22Z" name: isvc-sklearn-batcher-predictor.18c05e4d09e32880 namespace: kserve-ci-e2e-test resourceVersion: "15334" uid: 8696ebce-efd8-4ad3-bac9-fb4a85421e91 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-08T16:44:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "14628" uid: 43cc5539-5198-4070-8555-2238a2ddd229 kind: Event lastTimestamp: "2026-07-08T16:46:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:46:31Z" name: isvc-sklearn-batcher.18c05e428b3c8960 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: d9bc2408-98ed-4695-8ade-5638d4cae991 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "14628" uid: 43cc5539-5198-4070-8555-2238a2ddd229 kind: Event lastTimestamp: "2026-07-08T16:44:52Z" message: 'Failed to update status for InferenceService "isvc-sklearn-batcher": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:44:52Z" name: isvc-sklearn-batcher.18c05e428b9d9926 namespace: kserve-ci-e2e-test resourceVersion: "14647" uid: df47c07a-6637-4f91-b5a2-3da86344f8fc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:44:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "14628" uid: 43cc5539-5198-4070-8555-2238a2ddd229 kind: Event lastTimestamp: "2026-07-08T16:44:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-batcher": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:44:52Z" name: isvc-sklearn-batcher.18c05e428b9dddf2 namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: b6141849-8eca-4294-b21d-3fa7a1fa88cb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:46:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-batcher namespace: kserve-ci-e2e-test resourceVersion: "15398" uid: 43cc5539-5198-4070-8555-2238a2ddd229 kind: Event lastTimestamp: "2026-07-08T16:46:31Z" message: InferenceService [isvc-sklearn-batcher] is Ready metadata: creationTimestamp: "2026-07-08T16:46:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:46:31Z" name: isvc-sklearn-batcher.18c05e5993e013fe namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: 4cd8ff70-5c57-4339-a9c5-127f6d6beef9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:48:12.754163Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-mqzx2 namespace: kserve-ci-e2e-test resourceVersion: "16276" uid: 873bbe5a-e8a1-420f-aa8f-84bcaf656116 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-predictor-6575bcfdfd-mqzx2 to ip-10-0-139-237.ec2.internal metadata: creationTimestamp: "2026-07-08T16:48:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:48:12Z" name: message-dumper-predictor-6575bcfdfd-mqzx2.18c05e71351db77d namespace: kserve-ci-e2e-test resourceVersion: "16280" uid: d1d431d7-c61f-471a-8cc6-e8b94a0c3dea reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-867dc68758-8x97b source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:13Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-predictor-6575bcfdfd-mqzx2 namespace: kserve-ci-e2e-test resourceVersion: "16286" uid: 873bbe5a-e8a1-420f-aa8f-84bcaf656116 kind: Event lastTimestamp: "2026-07-08T16:48:13Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:48:13Z" name: message-dumper-predictor-6575bcfdfd-mqzx2.18c05e714f159770 namespace: kserve-ci-e2e-test resourceVersion: "16292" uid: b2f63196-74aa-4f5b-8442-c8f754c53bb5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-mqzx2 namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: 873bbe5a-e8a1-420f-aa8f-84bcaf656116 kind: Event lastTimestamp: "2026-07-08T16:48:13Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-08T16:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:13Z" name: message-dumper-predictor-6575bcfdfd-mqzx2.18c05e7150cf81de namespace: kserve-ci-e2e-test resourceVersion: "16294" uid: 5a937a59-4e35-4311-bf7f-c0528c43c818 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-mqzx2 namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: 873bbe5a-e8a1-420f-aa8f-84bcaf656116 kind: Event lastTimestamp: "2026-07-08T16:48:14Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.168s (1.168s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-08T16:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:14Z" name: message-dumper-predictor-6575bcfdfd-mqzx2.18c05e7196769c3f namespace: kserve-ci-e2e-test resourceVersion: "16301" uid: 66cabdfd-9187-4c3c-a85b-3446c2b7fa40 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-mqzx2 namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: 873bbe5a-e8a1-420f-aa8f-84bcaf656116 kind: Event lastTimestamp: "2026-07-08T16:48:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:14Z" name: message-dumper-predictor-6575bcfdfd-mqzx2.18c05e71998af786 namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: b601e47d-55d0-4047-af20-a797f5bf75c0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-mqzx2 namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: 873bbe5a-e8a1-420f-aa8f-84bcaf656116 kind: Event lastTimestamp: "2026-07-08T16:48:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:14Z" name: message-dumper-predictor-6575bcfdfd-mqzx2.18c05e719a047798 namespace: kserve-ci-e2e-test resourceVersion: "16304" uid: 496c17e0-ae51-4e5f-980e-c745e3d79797 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-mqzx2 namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: 873bbe5a-e8a1-420f-aa8f-84bcaf656116 kind: Event lastTimestamp: "2026-07-08T16:48:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:14Z" name: message-dumper-predictor-6575bcfdfd-mqzx2.18c05e719a0e7f77 namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: fd9b0044-b49d-487b-ad74-78d670706953 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-mqzx2 namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: 873bbe5a-e8a1-420f-aa8f-84bcaf656116 kind: Event lastTimestamp: "2026-07-08T16:48:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:14Z" name: message-dumper-predictor-6575bcfdfd-mqzx2.18c05e719d5883a4 namespace: kserve-ci-e2e-test resourceVersion: "16306" uid: 03b6bc8d-0444-4d70-8b40-44f23b8f03b3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-mqzx2 namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: 873bbe5a-e8a1-420f-aa8f-84bcaf656116 kind: Event lastTimestamp: "2026-07-08T16:48:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:48:14Z" name: message-dumper-predictor-6575bcfdfd-mqzx2.18c05e719dce58bd namespace: kserve-ci-e2e-test resourceVersion: "16307" uid: 7d624e67-1d55-4a12-8031-c2ee3fed2954 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-predictor-6575bcfdfd-mqzx2 namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: 873bbe5a-e8a1-420f-aa8f-84bcaf656116 kind: Event lastTimestamp: "2026-07-08T16:50:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:08Z" name: message-dumper-predictor-6575bcfdfd-mqzx2.18c05e8c145bee12 namespace: kserve-ci-e2e-test resourceVersion: "17288" uid: 5a1c2db0-4ad1-4e4f-b5b0-ed2e11299d59 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-predictor-6575bcfdfd-mqzx2 namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: 873bbe5a-e8a1-420f-aa8f-84bcaf656116 kind: Event lastTimestamp: "2026-07-08T16:50:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:50:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:08Z" name: message-dumper-predictor-6575bcfdfd-mqzx2.18c05e8c14601e25 namespace: kserve-ci-e2e-test resourceVersion: "17290" uid: 7839c047-f53c-4794-932b-26db9446f810 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-237.ec2.internal source: component: kubelet host: ip-10-0-139-237.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-predictor-6575bcfdfd namespace: kserve-ci-e2e-test resourceVersion: "16260" uid: b21c6433-625b-4ef2-924b-e13ebd53278c kind: Event lastTimestamp: "2026-07-08T16:48:12Z" message: 'Created pod: message-dumper-predictor-6575bcfdfd-mqzx2' metadata: creationTimestamp: "2026-07-08T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:12Z" name: message-dumper-predictor-6575bcfdfd.18c05e7134a7b95a namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 8868801f-a0e3-4485-8510-2e6c19d6320d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: 11bc0ef8-f96f-4247-9f98-57f7e78edfa9 kind: Event lastTimestamp: "2026-07-08T16:48:12Z" message: Scaled up replica set message-dumper-predictor-6575bcfdfd from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:12Z" name: message-dumper-predictor.18c05e7130977fe2 namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: 605d0f30-e7bb-46f5-9975-68acfde26df9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:48:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16258" uid: b7400823-a5af-4656-9582-1668b1be7954 kind: Event lastTimestamp: "2026-07-08T16:48:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:57Z" name: message-dumper-predictor.18c05e74af042642 namespace: kserve-ci-e2e-test resourceVersion: "16712" uid: f3492b5b-5b66-4ee4-b3d6-b50ef93830df reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:48:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16258" uid: b7400823-a5af-4656-9582-1668b1be7954 kind: Event lastTimestamp: "2026-07-08T16:48:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:57Z" name: message-dumper-predictor.18c05e74af6c867e namespace: kserve-ci-e2e-test resourceVersion: "16713" uid: a0e63a3b-7e2d-439b-a276-d4eef89a128d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16427" uid: b7400823-a5af-4656-9582-1668b1be7954 kind: Event lastTimestamp: "2026-07-08T16:48:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:48:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:42Z" name: message-dumper-predictor.18c05e782eb4536a namespace: kserve-ci-e2e-test resourceVersion: "16595" uid: 79703b00-a012-40fe-b759-dfc78eef94cb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-predictor namespace: kserve-ci-e2e-test resourceVersion: "16427" uid: b7400823-a5af-4656-9582-1668b1be7954 kind: Event lastTimestamp: "2026-07-08T16:48:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:48:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:48:42Z" name: message-dumper-predictor.18c05e782f1f0b95 namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: 64ff06f9-c1d8-475a-9792-2c0fbf3f04b2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:48:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 79533444-a191-4840-a058-766288620cbd kind: Event lastTimestamp: "2026-07-08T16:48:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:48:23Z" name: message-dumper.18c05e7130924c7e namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 6ff63758-0900-4cb3-ae0d-0c75b3bd28c6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 79533444-a191-4840-a058-766288620cbd kind: Event lastTimestamp: "2026-07-08T16:48:12Z" message: 'Failed to update status for InferenceService "message-dumper": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:48:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:48:12Z" name: message-dumper.18c05e71316ebda2 namespace: kserve-ci-e2e-test resourceVersion: "16267" uid: bbf018dc-dc63-4aa8-be6a-ab51dc0c30a1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:48:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper namespace: kserve-ci-e2e-test resourceVersion: "16391" uid: 79533444-a191-4840-a058-766288620cbd kind: Event lastTimestamp: "2026-07-08T16:48:23Z" message: InferenceService [message-dumper] is Ready metadata: creationTimestamp: "2026-07-08T16:48:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:48:23Z" name: message-dumper.18c05e739ae9f7f3 namespace: kserve-ci-e2e-test resourceVersion: "16392" uid: 3ad60213-1be2-4d59-bf7c-bf37cedab009 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "21839"