--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:35:35.576191Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-b37114-predictor-58fbffd4f5-x6r5k namespace: kserve-ci-e2e-test resourceVersion: "19244" uid: f8a7d276-9a15-40a2-975e-bd7d4797e2ae kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-b37114-predictor-58fbffd4f5-x6r5k to ip-10-0-129-161.ec2.internal metadata: creationTimestamp: "2026-04-22T14:35:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:35:35Z" name: isvc-init-fail-b37114-predictor-58fbffd4f5-x6r5k.18a8b484d3968aa4 namespace: kserve-ci-e2e-test resourceVersion: "19250" uid: bb8d6613-7951-4bb1-8e41-6966fb34835f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56d6f9bdf8-db99f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:35Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-b37114-predictor-58fbffd4f5-x6r5k namespace: kserve-ci-e2e-test resourceVersion: "19255" uid: f8a7d276-9a15-40a2-975e-bd7d4797e2ae kind: Event lastTimestamp: "2026-04-22T14:35:35Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:35:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:35:35Z" name: isvc-init-fail-b37114-predictor-58fbffd4f5-x6r5k.18a8b484ec4e9f42 namespace: kserve-ci-e2e-test resourceVersion: "19268" uid: 2618f592-50cf-431e-afcf-1f779e7d5b89 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-b37114-predictor-58fbffd4f5-x6r5k namespace: kserve-ci-e2e-test resourceVersion: "19246" uid: f8a7d276-9a15-40a2-975e-bd7d4797e2ae kind: Event lastTimestamp: "2026-04-22T14:35:41Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:498563386b50b61380307187059de966bc0a117e7161dd79d36731f0b2ebd43d" already present on machine metadata: creationTimestamp: "2026-04-22T14:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:41Z" name: isvc-init-fail-b37114-predictor-58fbffd4f5-x6r5k.18a8b484ed7dc99a namespace: kserve-ci-e2e-test resourceVersion: "19323" uid: 9c8f21d3-c385-4fa1-950e-7287b40728c1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-b37114-predictor-58fbffd4f5-x6r5k namespace: kserve-ci-e2e-test resourceVersion: "19246" uid: f8a7d276-9a15-40a2-975e-bd7d4797e2ae kind: Event lastTimestamp: "2026-04-22T14:35:41Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T14:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:41Z" name: isvc-init-fail-b37114-predictor-58fbffd4f5-x6r5k.18a8b484f06fac61 namespace: kserve-ci-e2e-test resourceVersion: "19327" uid: 0475d5b5-1a98-48ed-9341-20fde3bc710a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:35:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-b37114-predictor-58fbffd4f5-x6r5k namespace: kserve-ci-e2e-test resourceVersion: "19246" uid: f8a7d276-9a15-40a2-975e-bd7d4797e2ae kind: Event lastTimestamp: "2026-04-22T14:35:41Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T14:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:41Z" name: isvc-init-fail-b37114-predictor-58fbffd4f5-x6r5k.18a8b484f0dd0c5d namespace: kserve-ci-e2e-test resourceVersion: "19328" uid: f03ade63-4599-44ec-a82a-0e4a60a3d5b4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-b37114-predictor-58fbffd4f5-x6r5k namespace: kserve-ci-e2e-test resourceVersion: "19246" uid: f8a7d276-9a15-40a2-975e-bd7d4797e2ae kind: Event lastTimestamp: "2026-04-22T14:35:45Z" message: Back-off restarting failed container storage-initializer in pod isvc-init-fail-b37114-predictor-58fbffd4f5-x6r5k_kserve-ci-e2e-test(f8a7d276-9a15-40a2-975e-bd7d4797e2ae) metadata: creationTimestamp: "2026-04-22T14:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:45Z" name: isvc-init-fail-b37114-predictor-58fbffd4f5-x6r5k.18a8b4870c927fe3 namespace: kserve-ci-e2e-test resourceVersion: "19357" uid: 4fb05c2d-811f-4bd6-821e-ad498dea083f reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-b37114-predictor-58fbffd4f5 namespace: kserve-ci-e2e-test resourceVersion: "19236" uid: fa861cd5-24e1-4f25-a108-b06041f2b734 kind: Event lastTimestamp: "2026-04-22T14:35:35Z" message: 'Created pod: isvc-init-fail-b37114-predictor-58fbffd4f5-x6r5k' metadata: creationTimestamp: "2026-04-22T14:35:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:35:35Z" name: isvc-init-fail-b37114-predictor-58fbffd4f5.18a8b484d3273cde namespace: kserve-ci-e2e-test resourceVersion: "19245" uid: 8bed1f39-1d49-4e3f-92d9-8875458f739c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-b37114-predictor namespace: kserve-ci-e2e-test resourceVersion: "19233" uid: 09240440-a850-4053-8849-8f3c70e9181e kind: Event lastTimestamp: "2026-04-22T14:35:35Z" message: Scaled up replica set isvc-init-fail-b37114-predictor-58fbffd4f5 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:35:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:35:35Z" name: isvc-init-fail-b37114-predictor.18a8b484d0ad7ecb namespace: kserve-ci-e2e-test resourceVersion: "19238" uid: 9cff77ce-04cb-49ca-968e-76f5eae7c49d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-22T14:35:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-b37114 namespace: kserve-ci-e2e-test resourceVersion: "19211" uid: f89ae468-f205-4268-af4d-0fd0019029c5 kind: Event lastTimestamp: "2026-04-22T14:35:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:35:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:35:45Z" name: isvc-init-fail-b37114.18a8b484d08395eb namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: 98e7373b-3f2d-4981-b8ad-15fcd0f6d7f9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:35:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-b37114 namespace: kserve-ci-e2e-test resourceVersion: "19211" uid: f89ae468-f205-4268-af4d-0fd0019029c5 kind: Event lastTimestamp: "2026-04-22T14:35:35Z" message: 'Failed to update status for InferenceService "isvc-init-fail-b37114": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-b37114": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:35:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:35:35Z" name: isvc-init-fail-b37114.18a8b484d0cbfe8f namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: 57d9c8f3-6225-44bf-a8cc-ce0611ebc2b4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:35:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-b37114 namespace: kserve-ci-e2e-test resourceVersion: "19211" uid: f89ae468-f205-4268-af4d-0fd0019029c5 kind: Event lastTimestamp: "2026-04-22T14:35:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-b37114": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:35:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:35:35Z" name: isvc-init-fail-b37114.18a8b484d0cc5340 namespace: kserve-ci-e2e-test resourceVersion: "19258" uid: 55015334-0474-4622-a469-eb2036e8b4ea reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:30:01.800623Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5 namespace: kserve-ci-e2e-test resourceVersion: "16507" uid: 0b34a83e-7a85-410d-94ec-cf0e565f428f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5 to ip-10-0-129-161.ec2.internal metadata: creationTimestamp: "2026-04-22T14:30:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:30:01Z" name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5.18a8b4371d039254 namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: 65277403-7966-4b02-b03e-df06943e93cb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56d6f9bdf8-db99f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:30:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5 namespace: kserve-ci-e2e-test resourceVersion: "16518" uid: 0b34a83e-7a85-410d-94ec-cf0e565f428f kind: Event lastTimestamp: "2026-04-22T14:30:02Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:30:02Z" name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5.18a8b4373725a662 namespace: kserve-ci-e2e-test resourceVersion: "16525" uid: 237ca84d-e82d-4f55-afb0-ebb1b6aa9a39 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:30:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 0b34a83e-7a85-410d-94ec-cf0e565f428f kind: Event lastTimestamp: "2026-04-22T14:30:02Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:498563386b50b61380307187059de966bc0a117e7161dd79d36731f0b2ebd43d" already present on machine metadata: creationTimestamp: "2026-04-22T14:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:30:02Z" name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5.18a8b43738873832 namespace: kserve-ci-e2e-test resourceVersion: "16527" uid: 6b8b46b9-c9d7-4c4c-9cba-1d586330d358 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:30:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 0b34a83e-7a85-410d-94ec-cf0e565f428f kind: Event lastTimestamp: "2026-04-22T14:30:02Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T14:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:30:02Z" name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5.18a8b4373c98ef85 namespace: kserve-ci-e2e-test resourceVersion: "16529" uid: d7403a95-206b-4320-90d3-153f1bc5b009 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:30:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 0b34a83e-7a85-410d-94ec-cf0e565f428f kind: Event lastTimestamp: "2026-04-22T14:30:02Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T14:30:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:30:02Z" name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5.18a8b4373d0c8864 namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: 4a78995b-5c26-4536-96e2-135bbea42163 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 0b34a83e-7a85-410d-94ec-cf0e565f428f kind: Event lastTimestamp: "2026-04-22T14:30:07Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:30:07Z" name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5.18a8b4385d9ba70a namespace: kserve-ci-e2e-test resourceVersion: "16575" uid: 28b8c1eb-fb18-4e08-a97e-4458373a2b00 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 0b34a83e-7a85-410d-94ec-cf0e565f428f kind: Event lastTimestamp: "2026-04-22T14:30:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:30:07Z" name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5.18a8b43860a9da9e namespace: kserve-ci-e2e-test resourceVersion: "16580" uid: 523f8030-7775-4f54-92d6-99bf9eb798ee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 0b34a83e-7a85-410d-94ec-cf0e565f428f kind: Event lastTimestamp: "2026-04-22T14:30:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:30:07Z" name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5.18a8b438611c6b0e namespace: kserve-ci-e2e-test resourceVersion: "16581" uid: 4144826e-4a24-4207-8d36-b4d181eb36e4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 0b34a83e-7a85-410d-94ec-cf0e565f428f kind: Event lastTimestamp: "2026-04-22T14:30:07Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:62925d7b981d1952f65351ff12d2c91532437425fa51505583c8531f544bac1b" already present on machine metadata: creationTimestamp: "2026-04-22T14:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:30:07Z" name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5.18a8b43861284973 namespace: kserve-ci-e2e-test resourceVersion: "16582" uid: c385dc3e-e1e5-456d-aa7a-b73347d28125 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 0b34a83e-7a85-410d-94ec-cf0e565f428f kind: Event lastTimestamp: "2026-04-22T14:30:07Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-22T14:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:30:07Z" name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5.18a8b43863e8b0e1 namespace: kserve-ci-e2e-test resourceVersion: "16584" uid: 0bc96524-520c-4349-a57e-326c0519ed3e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 0b34a83e-7a85-410d-94ec-cf0e565f428f kind: Event lastTimestamp: "2026-04-22T14:30:07Z" message: Started container agent metadata: creationTimestamp: "2026-04-22T14:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:30:07Z" name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5.18a8b438645641d9 namespace: kserve-ci-e2e-test resourceVersion: "16585" uid: 4429db42-cf67-4cc8-a3c5-f703e16f456b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-22T14:30:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 0b34a83e-7a85-410d-94ec-cf0e565f428f kind: Event lastTimestamp: "2026-04-22T14:31:49Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:30:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:31:49Z" name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5.18a8b43899851846 namespace: kserve-ci-e2e-test resourceVersion: "17432" uid: 30a77f92-8abb-4f4a-bd48-e356b8a23929 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-22T14:30:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 0b34a83e-7a85-410d-94ec-cf0e565f428f kind: Event lastTimestamp: "2026-04-22T14:31:49Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T14:30:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:31:49Z" name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5.18a8b438998e939d namespace: kserve-ci-e2e-test resourceVersion: "17433" uid: 4be171bd-a350-477a-947f-7a9d654fba47 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 0b34a83e-7a85-410d-94ec-cf0e565f428f kind: Event lastTimestamp: "2026-04-22T14:31:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:31:26Z" name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5.18a8b44af15ccda5 namespace: kserve-ci-e2e-test resourceVersion: "17212" uid: d9096ed5-d455-4761-8be4-08a51524d1a9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:31:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: 0b34a83e-7a85-410d-94ec-cf0e565f428f kind: Event lastTimestamp: "2026-04-22T14:31:26Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-22T14:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:31:26Z" name: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5.18a8b44af15dd661 namespace: kserve-ci-e2e-test resourceVersion: "17214" uid: 44c1d694-da27-433a-b590-f568f93c9758 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:30:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-8d0fa-predictor-57bf898c67 namespace: kserve-ci-e2e-test resourceVersion: "16495" uid: 4309c296-302b-4b66-abfe-213c90974c06 kind: Event lastTimestamp: "2026-04-22T14:30:01Z" message: 'Created pod: isvc-logger-raw-8d0fa-predictor-57bf898c67-ffsb5' metadata: creationTimestamp: "2026-04-22T14:30:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:30:01Z" name: isvc-logger-raw-8d0fa-predictor-57bf898c67.18a8b4371c911080 namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: cd1cf75d-49fb-4ab8-be93-4683e4b0b6dd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:30:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-8d0fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: d3e437a7-5898-4151-a028-9cb04201fcbe kind: Event lastTimestamp: "2026-04-22T14:30:01Z" message: Scaled up replica set isvc-logger-raw-8d0fa-predictor-57bf898c67 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:30:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:30:01Z" name: isvc-logger-raw-8d0fa-predictor.18a8b43718ba83fd namespace: kserve-ci-e2e-test resourceVersion: "16497" uid: 676ccb84-88e1-4406-915b-ad57bfd9be8a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:30:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-8d0fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "16494" uid: dff51a2f-5c22-4749-bef9-adce7d4db718 kind: Event lastTimestamp: "2026-04-22T14:30:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:30:31Z" name: isvc-logger-raw-8d0fa-predictor.18a8b43a97829725 namespace: kserve-ci-e2e-test resourceVersion: "16768" uid: 010b9741-036d-46e4-879b-79a5c5b3b4d1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:30:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-8d0fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "16494" uid: dff51a2f-5c22-4749-bef9-adce7d4db718 kind: Event lastTimestamp: "2026-04-22T14:30:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:30:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:30:31Z" name: isvc-logger-raw-8d0fa-predictor.18a8b43a982200a8 namespace: kserve-ci-e2e-test resourceVersion: "16769" uid: 1af65b85-e212-4cb2-9d72-ec5fa5b85047 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:30:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-8d0fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "16652" uid: dff51a2f-5c22-4749-bef9-adce7d4db718 kind: Event lastTimestamp: "2026-04-22T14:31:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:30:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:31:16Z" name: isvc-logger-raw-8d0fa-predictor.18a8b441962212cc namespace: kserve-ci-e2e-test resourceVersion: "17098" uid: 2a5ba0bf-5454-4615-8d12-ae4d8c1d0170 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:30:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-8d0fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "16652" uid: dff51a2f-5c22-4749-bef9-adce7d4db718 kind: Event lastTimestamp: "2026-04-22T14:31:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:30:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:31:16Z" name: isvc-logger-raw-8d0fa-predictor.18a8b441969141aa namespace: kserve-ci-e2e-test resourceVersion: "17099" uid: d4ba3bdf-84b1-4028-89e5-7703ebc90a3b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-22T14:30:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-8d0fa namespace: kserve-ci-e2e-test resourceVersion: "16484" uid: 02d299d5-a20f-4ff9-955a-8f6a24b6a98a kind: Event lastTimestamp: "2026-04-22T14:31:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:30:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:31:19Z" name: isvc-logger-raw-8d0fa.18a8b437186af9a5 namespace: kserve-ci-e2e-test resourceVersion: "17129" uid: d6bcfa3f-e45e-41c6-844d-db8f14521cc7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:30:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-8d0fa namespace: kserve-ci-e2e-test resourceVersion: "16484" uid: 02d299d5-a20f-4ff9-955a-8f6a24b6a98a kind: Event lastTimestamp: "2026-04-22T14:30:01Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-8d0fa": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-8d0fa": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:30:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:30:01Z" name: isvc-logger-raw-8d0fa.18a8b43719504964 namespace: kserve-ci-e2e-test resourceVersion: "16519" uid: c48c06fc-0ed5-4e8d-b274-66ac76d1e55d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:30:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-8d0fa namespace: kserve-ci-e2e-test resourceVersion: "16493" uid: 02d299d5-a20f-4ff9-955a-8f6a24b6a98a kind: Event lastTimestamp: "2026-04-22T14:30:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-8d0fa": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:30:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:30:01Z" name: isvc-logger-raw-8d0fa.18a8b4371e8a1c56 namespace: kserve-ci-e2e-test resourceVersion: "16520" uid: 496ad4a4-e08e-41cc-98ed-d9c13c05392f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:31:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-8d0fa namespace: kserve-ci-e2e-test resourceVersion: "17127" uid: 02d299d5-a20f-4ff9-955a-8f6a24b6a98a kind: Event lastTimestamp: "2026-04-22T14:31:19Z" message: InferenceService [isvc-logger-raw-8d0fa] is Ready metadata: creationTimestamp: "2026-04-22T14:31:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:31:19Z" name: isvc-logger-raw-8d0fa.18a8b4493fde5272 namespace: kserve-ci-e2e-test resourceVersion: "17128" uid: 1a5258a3-fa34-4760-a853-af62c2ec2200 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:33:57.280038Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc namespace: kserve-ci-e2e-test resourceVersion: "18380" uid: f390d180-b8f4-4f2d-b76a-f9178bf4d3ff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc to ip-10-0-129-161.ec2.internal metadata: creationTimestamp: "2026-04-22T14:33:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:33:57Z" name: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc.18a8b46df0ae327d namespace: kserve-ci-e2e-test resourceVersion: "18386" uid: aa639c11-ca29-48a7-9809-56e963dbc242 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56d6f9bdf8-db99f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:33:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc namespace: kserve-ci-e2e-test resourceVersion: "18391" uid: f390d180-b8f4-4f2d-b76a-f9178bf4d3ff kind: Event lastTimestamp: "2026-04-22T14:33:57Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:33:57Z" name: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc.18a8b46e09738773 namespace: kserve-ci-e2e-test resourceVersion: "18398" uid: 6206bab9-6604-431d-9836-c2bc799f5150 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:33:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc namespace: kserve-ci-e2e-test resourceVersion: "18383" uid: f390d180-b8f4-4f2d-b76a-f9178bf4d3ff kind: Event lastTimestamp: "2026-04-22T14:33:57Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:498563386b50b61380307187059de966bc0a117e7161dd79d36731f0b2ebd43d" already present on machine metadata: creationTimestamp: "2026-04-22T14:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:33:57Z" name: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc.18a8b46e0a9b5c62 namespace: kserve-ci-e2e-test resourceVersion: "18400" uid: 23752d0b-9707-4297-a0fc-4dad1e321c9f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:33:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc namespace: kserve-ci-e2e-test resourceVersion: "18383" uid: f390d180-b8f4-4f2d-b76a-f9178bf4d3ff kind: Event lastTimestamp: "2026-04-22T14:33:57Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T14:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:33:57Z" name: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc.18a8b46e0d427615 namespace: kserve-ci-e2e-test resourceVersion: "18403" uid: 8c2d8eea-38a6-4018-85fe-18a1ed8ae695 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:33:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc namespace: kserve-ci-e2e-test resourceVersion: "18383" uid: f390d180-b8f4-4f2d-b76a-f9178bf4d3ff kind: Event lastTimestamp: "2026-04-22T14:33:57Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T14:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:33:57Z" name: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc.18a8b46e0daf451d namespace: kserve-ci-e2e-test resourceVersion: "18404" uid: 5a53f166-2691-4a29-aec4-673632b458bb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc namespace: kserve-ci-e2e-test resourceVersion: "18383" uid: f390d180-b8f4-4f2d-b76a-f9178bf4d3ff kind: Event lastTimestamp: "2026-04-22T14:34:01Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:34:01Z" name: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc.18a8b46eff31b014 namespace: kserve-ci-e2e-test resourceVersion: "18442" uid: 83796d12-dadb-481c-8d60-83861a623990 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc namespace: kserve-ci-e2e-test resourceVersion: "18383" uid: f390d180-b8f4-4f2d-b76a-f9178bf4d3ff kind: Event lastTimestamp: "2026-04-22T14:34:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:34:01Z" name: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc.18a8b46f023a5910 namespace: kserve-ci-e2e-test resourceVersion: "18445" uid: ecf62dc5-8854-49fe-b1f1-f2caf0af2db0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc namespace: kserve-ci-e2e-test resourceVersion: "18383" uid: f390d180-b8f4-4f2d-b76a-f9178bf4d3ff kind: Event lastTimestamp: "2026-04-22T14:34:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:34:01Z" name: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc.18a8b46f02ac5e5e namespace: kserve-ci-e2e-test resourceVersion: "18447" uid: 9e46d82a-b084-4826-b739-c5f881602c9a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-22T14:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc namespace: kserve-ci-e2e-test resourceVersion: "18383" uid: f390d180-b8f4-4f2d-b76a-f9178bf4d3ff kind: Event lastTimestamp: "2026-04-22T14:35:03Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:03Z" name: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc.18a8b46f3b10746b namespace: kserve-ci-e2e-test resourceVersion: "18885" uid: 44ed6763-a9c6-49d1-bf67-614a8593691f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc namespace: kserve-ci-e2e-test resourceVersion: "18383" uid: f390d180-b8f4-4f2d-b76a-f9178bf4d3ff kind: Event lastTimestamp: "2026-04-22T14:35:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:35:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:35Z" name: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc.18a8b484cb2eb2c4 namespace: kserve-ci-e2e-test resourceVersion: "19208" uid: dcfd86a1-4204-4f15-8073-7c9d5ddf690f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:33:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-d7c1cc-predictor-85f6c6b6f6 namespace: kserve-ci-e2e-test resourceVersion: "18368" uid: a7697cef-ffdc-4ec7-8bfe-3b468f845cf3 kind: Event lastTimestamp: "2026-04-22T14:33:57Z" message: 'Created pod: isvc-primary-d7c1cc-predictor-85f6c6b6f6-9j4hc' metadata: creationTimestamp: "2026-04-22T14:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:33:57Z" name: isvc-primary-d7c1cc-predictor-85f6c6b6f6.18a8b46df03fc0f5 namespace: kserve-ci-e2e-test resourceVersion: "18382" uid: c2f85c69-f730-45fd-bdea-3f35b9b3f93e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:33:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-d7c1cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18363" uid: e7579a93-3e54-429f-9110-bf7edbe6ee93 kind: Event lastTimestamp: "2026-04-22T14:33:57Z" message: Scaled up replica set isvc-primary-d7c1cc-predictor-85f6c6b6f6 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:33:57Z" name: isvc-primary-d7c1cc-predictor.18a8b46dec6bd6e5 namespace: kserve-ci-e2e-test resourceVersion: "18370" uid: cbc87829-5e4a-4b6e-9423-15df0de008cb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:34:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-d7c1cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18367" uid: 1c579e5d-964f-4d47-8a0b-0028e3cefd0f kind: Event lastTimestamp: "2026-04-22T14:34:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:34:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:34:42Z" name: isvc-primary-d7c1cc-predictor.18a8b4716b5e183f namespace: kserve-ci-e2e-test resourceVersion: "18738" uid: 52337b5f-cbc7-436d-bbe4-e9aa4cd536df reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:34:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-d7c1cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18367" uid: 1c579e5d-964f-4d47-8a0b-0028e3cefd0f kind: Event lastTimestamp: "2026-04-22T14:34:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:34:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:34:42Z" name: isvc-primary-d7c1cc-predictor.18a8b4716bc6c157 namespace: kserve-ci-e2e-test resourceVersion: "18739" uid: 011c1589-215b-4194-aa0b-4fa8eadd17ec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:34:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-d7c1cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18523" uid: 1c579e5d-964f-4d47-8a0b-0028e3cefd0f kind: Event lastTimestamp: "2026-04-22T14:35:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:34:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:35:27Z" name: isvc-primary-d7c1cc-predictor.18a8b47be8874627 namespace: kserve-ci-e2e-test resourceVersion: "19117" uid: e7d7dbe8-a547-41c3-a957-2481cb06fe7d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:34:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-d7c1cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "18523" uid: 1c579e5d-964f-4d47-8a0b-0028e3cefd0f kind: Event lastTimestamp: "2026-04-22T14:35:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:34:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:35:27Z" name: isvc-primary-d7c1cc-predictor.18a8b47be8ef5608 namespace: kserve-ci-e2e-test resourceVersion: "19118" uid: 4daeddc4-51b2-4db4-ba0b-d3b6d5617c48 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-22T14:33:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-d7c1cc namespace: kserve-ci-e2e-test resourceVersion: "18342" uid: 22cd0fe2-2039-4bc3-9aab-54fcdd01c5ee kind: Event lastTimestamp: "2026-04-22T14:35:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:35:13Z" name: isvc-primary-d7c1cc.18a8b46dec6a91dd namespace: kserve-ci-e2e-test resourceVersion: "18969" uid: 3cb32a07-b9e1-4897-810f-3f7679f276a2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:33:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-d7c1cc namespace: kserve-ci-e2e-test resourceVersion: "18342" uid: 22cd0fe2-2039-4bc3-9aab-54fcdd01c5ee kind: Event lastTimestamp: "2026-04-22T14:33:57Z" message: 'Failed to update status for InferenceService "isvc-primary-d7c1cc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-d7c1cc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:33:57Z" name: isvc-primary-d7c1cc.18a8b46ded2879c0 namespace: kserve-ci-e2e-test resourceVersion: "18389" uid: dd564892-db87-42ff-b01e-6db65814c5cb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:33:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-d7c1cc namespace: kserve-ci-e2e-test resourceVersion: "18342" uid: 22cd0fe2-2039-4bc3-9aab-54fcdd01c5ee kind: Event lastTimestamp: "2026-04-22T14:33:57Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-d7c1cc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:33:57Z" name: isvc-primary-d7c1cc.18a8b46ded28b726 namespace: kserve-ci-e2e-test resourceVersion: "18392" uid: ad41aca2-0139-43cd-87f6-27cce44c34f6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-d7c1cc namespace: kserve-ci-e2e-test resourceVersion: "18967" uid: 22cd0fe2-2039-4bc3-9aab-54fcdd01c5ee kind: Event lastTimestamp: "2026-04-22T14:35:13Z" message: InferenceService [isvc-primary-d7c1cc] is Ready metadata: creationTimestamp: "2026-04-22T14:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:35:13Z" name: isvc-primary-d7c1cc.18a8b47fc7405647 namespace: kserve-ci-e2e-test resourceVersion: "18968" uid: 562ee8f7-f0f8-415a-b468-f4f9a70eec51 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:24:55.855516Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh namespace: kserve-ci-e2e-test resourceVersion: "13673" uid: 65512743-a88c-4def-8691-ddc0f0febb55 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh to ip-10-0-129-161.ec2.internal metadata: creationTimestamp: "2026-04-22T14:24:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:24:55Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh.18a8b3efe143bc84 namespace: kserve-ci-e2e-test resourceVersion: "13679" uid: e0a5a949-5396-4e8d-a598-44c75b3644b3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56d6f9bdf8-db99f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh namespace: kserve-ci-e2e-test resourceVersion: "13684" uid: 65512743-a88c-4def-8691-ddc0f0febb55 kind: Event lastTimestamp: "2026-04-22T14:24:56Z" message: Add eth0 [10.132.0.16/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:24:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:24:56Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh.18a8b3effa9aaf12 namespace: kserve-ci-e2e-test resourceVersion: "13694" uid: 1bb01277-c8b9-4d46-86e2-5128619bc6ec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: 65512743-a88c-4def-8691-ddc0f0febb55 kind: Event lastTimestamp: "2026-04-22T14:24:56Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:498563386b50b61380307187059de966bc0a117e7161dd79d36731f0b2ebd43d" metadata: creationTimestamp: "2026-04-22T14:24:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:56Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh.18a8b3effc04c51c namespace: kserve-ci-e2e-test resourceVersion: "13696" uid: cd31dd85-f537-4d54-899a-7c1de35a5719 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: 65512743-a88c-4def-8691-ddc0f0febb55 kind: Event lastTimestamp: "2026-04-22T14:24:59Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:498563386b50b61380307187059de966bc0a117e7161dd79d36731f0b2ebd43d" in 3.328s (3.328s including waiting). Image size: 299663288 bytes.' metadata: creationTimestamp: "2026-04-22T14:24:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:59Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh.18a8b3f0c26cc658 namespace: kserve-ci-e2e-test resourceVersion: "13725" uid: 7754b3be-34e6-4403-b5bf-073cba450d35 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: 65512743-a88c-4def-8691-ddc0f0febb55 kind: Event lastTimestamp: "2026-04-22T14:24:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T14:24:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:59Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh.18a8b3f0c5d3849f namespace: kserve-ci-e2e-test resourceVersion: "13726" uid: a0144c92-9c42-408f-be94-c63c5abce36b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: 65512743-a88c-4def-8691-ddc0f0febb55 kind: Event lastTimestamp: "2026-04-22T14:24:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T14:24:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:24:59Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh.18a8b3f0c64434e3 namespace: kserve-ci-e2e-test resourceVersion: "13727" uid: 2a054b74-1741-4d8f-bee7-bb1efa01d53d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: 65512743-a88c-4def-8691-ddc0f0febb55 kind: Event lastTimestamp: "2026-04-22T14:25:04Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1400" metadata: creationTimestamp: "2026-04-22T14:25:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:25:04Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh.18a8b3f1d8cbfc2e namespace: kserve-ci-e2e-test resourceVersion: "13764" uid: a6ad4c73-73cf-4c4d-9568-c3acb4847c69 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: 65512743-a88c-4def-8691-ddc0f0febb55 kind: Event lastTimestamp: "2026-04-22T14:25:17Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1400" in 12.867s (12.867s including waiting). Image size: 1562514855 bytes.' metadata: creationTimestamp: "2026-04-22T14:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:25:17Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh.18a8b3f4d7c3f465 namespace: kserve-ci-e2e-test resourceVersion: "13861" uid: a09346ea-3c78-4aba-bcdd-58e3f8276027 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: 65512743-a88c-4def-8691-ddc0f0febb55 kind: Event lastTimestamp: "2026-04-22T14:25:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:25:17Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh.18a8b3f4dac7e3b3 namespace: kserve-ci-e2e-test resourceVersion: "13862" uid: 18cbba8e-1fa4-48c8-9aa3-40d82a59e074 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: 65512743-a88c-4def-8691-ddc0f0febb55 kind: Event lastTimestamp: "2026-04-22T14:25:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:25:17Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh.18a8b3f4db3af005 namespace: kserve-ci-e2e-test resourceVersion: "13863" uid: 5145e045-aa91-4190-a2ea-9b242789dfed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: 65512743-a88c-4def-8691-ddc0f0febb55 kind: Event lastTimestamp: "2026-04-22T14:25:17Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:62925d7b981d1952f65351ff12d2c91532437425fa51505583c8531f544bac1b" metadata: creationTimestamp: "2026-04-22T14:25:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:25:17Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh.18a8b3f4db46c80b namespace: kserve-ci-e2e-test resourceVersion: "13864" uid: fbd45f00-8a57-4aa2-bd27-c301f5d4d9b1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: 65512743-a88c-4def-8691-ddc0f0febb55 kind: Event lastTimestamp: "2026-04-22T14:25:20Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:62925d7b981d1952f65351ff12d2c91532437425fa51505583c8531f544bac1b" in 3.266s (3.266s including waiting). Image size: 237897788 bytes.' metadata: creationTimestamp: "2026-04-22T14:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:25:20Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh.18a8b3f59dfac38c namespace: kserve-ci-e2e-test resourceVersion: "13890" uid: b147ba97-f06d-427b-b480-5497b293683e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: 65512743-a88c-4def-8691-ddc0f0febb55 kind: Event lastTimestamp: "2026-04-22T14:25:20Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-22T14:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:25:20Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh.18a8b3f5a13911bf namespace: kserve-ci-e2e-test resourceVersion: "13892" uid: f352e480-489a-451e-98aa-d916f72a947e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:25:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: 65512743-a88c-4def-8691-ddc0f0febb55 kind: Event lastTimestamp: "2026-04-22T14:25:20Z" message: Started container agent metadata: creationTimestamp: "2026-04-22T14:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:25:20Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh.18a8b3f5a1a9ddfe namespace: kserve-ci-e2e-test resourceVersion: "13893" uid: b93c7d45-31b7-416e-88ed-3ac24b9ae5c2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T14:25:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: 65512743-a88c-4def-8691-ddc0f0febb55 kind: Event lastTimestamp: "2026-04-22T14:26:52Z" message: 'Readiness probe failed: dial tcp 10.132.0.16:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:52Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh.18a8b3f5d19d52eb namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 447db2fd-9fe2-48ec-8f4a-ecda0886b98f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T14:25:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: 65512743-a88c-4def-8691-ddc0f0febb55 kind: Event lastTimestamp: "2026-04-22T14:26:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T14:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:52Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh.18a8b3f5d1a643bb namespace: kserve-ci-e2e-test resourceVersion: "14698" uid: d37b9059-54b8-47e0-a297-fda563d896a0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: 65512743-a88c-4def-8691-ddc0f0febb55 kind: Event lastTimestamp: "2026-04-22T14:26:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:31Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh.18a8b4060b740202 namespace: kserve-ci-e2e-test resourceVersion: "14443" uid: 35203fb8-7d5b-4554-a95f-795f6001447a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: 65512743-a88c-4def-8691-ddc0f0febb55 kind: Event lastTimestamp: "2026-04-22T14:26:31Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-22T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:31Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh.18a8b4060b749d13 namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: c209e8af-fb5c-4b9e-974d-4a340a9b15c5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf namespace: kserve-ci-e2e-test resourceVersion: "13663" uid: 052c8c8d-c5de-4408-af74-d1a2d1627327 kind: Event lastTimestamp: "2026-04-22T14:24:55Z" message: 'Created pod: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf-tqwrh' metadata: creationTimestamp: "2026-04-22T14:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:24:55Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf.18a8b3efe0d01ec8 namespace: kserve-ci-e2e-test resourceVersion: "13675" uid: c8656de1-4a71-4855-895b-3de6b71a029b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-8cf9f-predictor namespace: kserve-ci-e2e-test resourceVersion: "13662" uid: 97a43a8e-d386-4051-af02-6d57e72cb6f2 kind: Event lastTimestamp: "2026-04-22T14:24:55Z" message: Scaled up replica set isvc-raw-sklearn-batcher-8cf9f-predictor-6cd474d9cf from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:24:55Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor.18a8b3efda96d020 namespace: kserve-ci-e2e-test resourceVersion: "13666" uid: 1c04e321-eb23-47e4-8f86-61120efb0e23 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T14:25:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-8cf9f-predictor namespace: kserve-ci-e2e-test resourceVersion: "13672" uid: 1c248ceb-d668-4c38-ac35-612ddc6b4688 kind: Event lastTimestamp: "2026-04-22T14:25:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:25:55Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor.18a8b3f36018fdcc namespace: kserve-ci-e2e-test resourceVersion: "14141" uid: 5f417175-ef14-4903-93ff-c5ff3390f8a3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T14:25:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-8cf9f-predictor namespace: kserve-ci-e2e-test resourceVersion: "13672" uid: 1c248ceb-d668-4c38-ac35-612ddc6b4688 kind: Event lastTimestamp: "2026-04-22T14:25:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:25:55Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor.18a8b3f3608a679c namespace: kserve-ci-e2e-test resourceVersion: "14142" uid: 5436e738-203e-4671-b955-88aed7fba63d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:26:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-8cf9f-predictor namespace: kserve-ci-e2e-test resourceVersion: "13818" uid: 1c248ceb-d668-4c38-ac35-612ddc6b4688 kind: Event lastTimestamp: "2026-04-22T14:26:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:25Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor.18a8b4015d6ce026 namespace: kserve-ci-e2e-test resourceVersion: "14368" uid: a5519e82-32c1-4eae-855d-eb464f44cb54 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:26:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-8cf9f-predictor namespace: kserve-ci-e2e-test resourceVersion: "13818" uid: 1c248ceb-d668-4c38-ac35-612ddc6b4688 kind: Event lastTimestamp: "2026-04-22T14:26:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:25Z" name: isvc-raw-sklearn-batcher-8cf9f-predictor.18a8b4015e073e7e namespace: kserve-ci-e2e-test resourceVersion: "14369" uid: 6d60ad14-b7c0-4065-b1c6-298b009b0a58 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-22T14:24:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-8cf9f namespace: kserve-ci-e2e-test resourceVersion: "13653" uid: 8cffd520-98c5-4f16-a512-3333b9a664ac kind: Event lastTimestamp: "2026-04-22T14:26:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:26:22Z" name: isvc-raw-sklearn-batcher-8cf9f.18a8b3efe0a2fb84 namespace: kserve-ci-e2e-test resourceVersion: "14344" uid: fbaa135d-077b-41c3-bb0d-6caaf841c3d4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-8cf9f namespace: kserve-ci-e2e-test resourceVersion: "13653" uid: 8cffd520-98c5-4f16-a512-3333b9a664ac kind: Event lastTimestamp: "2026-04-22T14:24:55Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-8cf9f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-8cf9f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:24:55Z" name: isvc-raw-sklearn-batcher-8cf9f.18a8b3efe10194a9 namespace: kserve-ci-e2e-test resourceVersion: "13678" uid: 559a9d18-40a5-4196-a324-9f3ad3630f2b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:24:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-8cf9f namespace: kserve-ci-e2e-test resourceVersion: "13653" uid: 8cffd520-98c5-4f16-a512-3333b9a664ac kind: Event lastTimestamp: "2026-04-22T14:24:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-8cf9f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:24:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:24:55Z" name: isvc-raw-sklearn-batcher-8cf9f.18a8b3efe101d5ed namespace: kserve-ci-e2e-test resourceVersion: "13680" uid: 12a0e659-7e78-4679-b33c-94c0e6f8b290 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-8cf9f namespace: kserve-ci-e2e-test resourceVersion: "14342" uid: 8cffd520-98c5-4f16-a512-3333b9a664ac kind: Event lastTimestamp: "2026-04-22T14:26:22Z" message: InferenceService [isvc-raw-sklearn-batcher-8cf9f] is Ready metadata: creationTimestamp: "2026-04-22T14:26:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:26:22Z" name: isvc-raw-sklearn-batcher-8cf9f.18a8b4040b190a14 namespace: kserve-ci-e2e-test resourceVersion: "14343" uid: 6e08a789-1b8b-4917-bfd7-8e0d5b60dc93 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:35:17.407431Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-d7c1cc-predictor-85c49f4765-kp75j namespace: kserve-ci-e2e-test resourceVersion: "19017" uid: 58d4b1dd-2bea-4a5c-a3bf-2d271dae167a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-d7c1cc-predictor-85c49f4765-kp75j to ip-10-0-129-161.ec2.internal metadata: creationTimestamp: "2026-04-22T14:35:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:35:17Z" name: isvc-secondary-d7c1cc-predictor-85c49f4765-kp75j.18a8b48098a5600a namespace: kserve-ci-e2e-test resourceVersion: "19021" uid: 03609a49-f01a-42c6-9fdf-8eff6bec4dac reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56d6f9bdf8-db99f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:17Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-d7c1cc-predictor-85c49f4765-kp75j namespace: kserve-ci-e2e-test resourceVersion: "19026" uid: 58d4b1dd-2bea-4a5c-a3bf-2d271dae167a kind: Event lastTimestamp: "2026-04-22T14:35:17Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:35:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:35:17Z" name: isvc-secondary-d7c1cc-predictor-85c49f4765-kp75j.18a8b480b190df34 namespace: kserve-ci-e2e-test resourceVersion: "19038" uid: ad7e0a67-e7b6-45a1-ba58-d5ae4a26a50f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:35:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-d7c1cc-predictor-85c49f4765-kp75j namespace: kserve-ci-e2e-test resourceVersion: "19018" uid: 58d4b1dd-2bea-4a5c-a3bf-2d271dae167a kind: Event lastTimestamp: "2026-04-22T14:35:24Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:498563386b50b61380307187059de966bc0a117e7161dd79d36731f0b2ebd43d" already present on machine metadata: creationTimestamp: "2026-04-22T14:35:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:24Z" name: isvc-secondary-d7c1cc-predictor-85c49f4765-kp75j.18a8b480b2db3b92 namespace: kserve-ci-e2e-test resourceVersion: "19089" uid: b7416b3c-9562-4b83-95ba-114136ed495d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:35:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-d7c1cc-predictor-85c49f4765-kp75j namespace: kserve-ci-e2e-test resourceVersion: "19018" uid: 58d4b1dd-2bea-4a5c-a3bf-2d271dae167a kind: Event lastTimestamp: "2026-04-22T14:35:24Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T14:35:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:24Z" name: isvc-secondary-d7c1cc-predictor-85c49f4765-kp75j.18a8b480b5797165 namespace: kserve-ci-e2e-test resourceVersion: "19091" uid: 584f6ba9-8968-43c6-ad45-5d27b714a8a3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:35:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-d7c1cc-predictor-85c49f4765-kp75j namespace: kserve-ci-e2e-test resourceVersion: "19018" uid: 58d4b1dd-2bea-4a5c-a3bf-2d271dae167a kind: Event lastTimestamp: "2026-04-22T14:35:24Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T14:35:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:24Z" name: isvc-secondary-d7c1cc-predictor-85c49f4765-kp75j.18a8b480b5eb5f81 namespace: kserve-ci-e2e-test resourceVersion: "19094" uid: fa0b2770-495d-42ed-9880-43ab5355f440 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-d7c1cc-predictor-85c49f4765-kp75j namespace: kserve-ci-e2e-test resourceVersion: "19018" uid: 58d4b1dd-2bea-4a5c-a3bf-2d271dae167a kind: Event lastTimestamp: "2026-04-22T14:35:30Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-d7c1cc-predictor-85c49f4765-kp75j_kserve-ci-e2e-test(58d4b1dd-2bea-4a5c-a3bf-2d271dae167a) metadata: creationTimestamp: "2026-04-22T14:35:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:30Z" name: isvc-secondary-d7c1cc-predictor-85c49f4765-kp75j.18a8b4838b93997e namespace: kserve-ci-e2e-test resourceVersion: "19139" uid: 104b1d00-6c2e-4b4a-8660-eabc924547cc reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-d7c1cc-predictor-85c49f4765 namespace: kserve-ci-e2e-test resourceVersion: "19008" uid: 8e8f38cb-b39c-4460-9668-f15f8a622163 kind: Event lastTimestamp: "2026-04-22T14:35:17Z" message: 'Created pod: isvc-secondary-d7c1cc-predictor-85c49f4765-kp75j' metadata: creationTimestamp: "2026-04-22T14:35:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:35:17Z" name: isvc-secondary-d7c1cc-predictor-85c49f4765.18a8b48098354a6a namespace: kserve-ci-e2e-test resourceVersion: "19019" uid: 5285fea2-81ec-4dca-beab-15c37a905f4a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-d7c1cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "19005" uid: cbadd08a-1a1b-4530-b49e-1ec308e42e3a kind: Event lastTimestamp: "2026-04-22T14:35:17Z" message: Scaled up replica set isvc-secondary-d7c1cc-predictor-85c49f4765 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:35:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:35:17Z" name: isvc-secondary-d7c1cc-predictor.18a8b4809545d071 namespace: kserve-ci-e2e-test resourceVersion: "19009" uid: e2eda423-45b0-4cea-82d4-3d7554314709 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-d7c1cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "19007" uid: 7531faa1-0ae9-460a-bc9c-d37493ae6b2a kind: Event lastTimestamp: "2026-04-22T14:35:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:35:32Z" name: isvc-secondary-d7c1cc-predictor.18a8b48413ea3601 namespace: kserve-ci-e2e-test resourceVersion: "19163" uid: a716ff8a-7a8e-4068-9f85-6a55c41c6c1e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-d7c1cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "19007" uid: 7531faa1-0ae9-460a-bc9c-d37493ae6b2a kind: Event lastTimestamp: "2026-04-22T14:35:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:35:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:35:32Z" name: isvc-secondary-d7c1cc-predictor.18a8b484145ff4e3 namespace: kserve-ci-e2e-test resourceVersion: "19165" uid: f0cffece-af07-4bdc-9d8d-7557c87146cf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-04-22T14:35:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-d7c1cc namespace: kserve-ci-e2e-test resourceVersion: "18997" uid: 11e97867-5466-4961-97d4-f88c1e022ebb kind: Event lastTimestamp: "2026-04-22T14:35:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:35:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:35:30Z" name: isvc-secondary-d7c1cc.18a8b480954b8dde namespace: kserve-ci-e2e-test resourceVersion: "19146" uid: cd540e5b-5daf-4bba-bcce-e74dcb589bf3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:35:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-d7c1cc namespace: kserve-ci-e2e-test resourceVersion: "18997" uid: 11e97867-5466-4961-97d4-f88c1e022ebb kind: Event lastTimestamp: "2026-04-22T14:35:17Z" message: 'Failed to update status for InferenceService "isvc-secondary-d7c1cc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-d7c1cc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:35:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:35:17Z" name: isvc-secondary-d7c1cc.18a8b48095dd76d9 namespace: kserve-ci-e2e-test resourceVersion: "19028" uid: bbb5a1ab-e8db-4504-8bfd-af4d4e768079 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:35:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-d7c1cc namespace: kserve-ci-e2e-test resourceVersion: "18997" uid: 11e97867-5466-4961-97d4-f88c1e022ebb kind: Event lastTimestamp: "2026-04-22T14:35:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-d7c1cc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:35:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:35:17Z" name: isvc-secondary-d7c1cc.18a8b48095ddaa83 namespace: kserve-ci-e2e-test resourceVersion: "19029" uid: 2e094e9d-3b07-4ae0-8aa5-42d080d716cb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:26:31.137219Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: b20ca612-2287-4a0f-a3fc-210e26f59ada kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx to ip-10-0-129-161.ec2.internal metadata: creationTimestamp: "2026-04-22T14:26:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:26:31Z" name: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx.18a8b406107f1e44 namespace: kserve-ci-e2e-test resourceVersion: "14463" uid: d62881bb-c611-444c-a285-e07a93a2b005 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56d6f9bdf8-db99f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: b20ca612-2287-4a0f-a3fc-210e26f59ada kind: Event lastTimestamp: "2026-04-22T14:26:31Z" message: Add eth0 [10.132.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:26:31Z" name: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx.18a8b40629966637 namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: 27fb3d99-b3af-419b-b0cf-03e68ad8a2f1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: b20ca612-2287-4a0f-a3fc-210e26f59ada kind: Event lastTimestamp: "2026-04-22T14:26:31Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:498563386b50b61380307187059de966bc0a117e7161dd79d36731f0b2ebd43d" already present on machine metadata: creationTimestamp: "2026-04-22T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:31Z" name: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx.18a8b4062b6fa439 namespace: kserve-ci-e2e-test resourceVersion: "14505" uid: 218be32c-2d42-4a26-bc97-d86dd85c0c2c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: b20ca612-2287-4a0f-a3fc-210e26f59ada kind: Event lastTimestamp: "2026-04-22T14:26:31Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:31Z" name: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx.18a8b4062ed3686e namespace: kserve-ci-e2e-test resourceVersion: "14507" uid: ff825773-eeb7-4575-a45b-c18d8f4383bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: b20ca612-2287-4a0f-a3fc-210e26f59ada kind: Event lastTimestamp: "2026-04-22T14:26:31Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:31Z" name: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx.18a8b4062f4f2f33 namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 1b0c6ad1-06f6-48b3-97ca-a8ed12279bd4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: b20ca612-2287-4a0f-a3fc-210e26f59ada kind: Event lastTimestamp: "2026-04-22T14:26:35Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:35Z" name: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx.18a8b407188a44a7 namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: 9cea5435-8b26-4b1d-9a35-1fd71e5630f2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: b20ca612-2287-4a0f-a3fc-210e26f59ada kind: Event lastTimestamp: "2026-04-22T14:26:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:35Z" name: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx.18a8b4071b90f053 namespace: kserve-ci-e2e-test resourceVersion: "14564" uid: d5fa5a15-c996-44d3-b21d-fcc81c5353ad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: b20ca612-2287-4a0f-a3fc-210e26f59ada kind: Event lastTimestamp: "2026-04-22T14:26:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:35Z" name: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx.18a8b4071bebe912 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 7ad58b8a-4526-48b4-95cc-fd53a185fd98 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-22T14:26:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: b20ca612-2287-4a0f-a3fc-210e26f59ada kind: Event lastTimestamp: "2026-04-22T14:28:15Z" message: 'Readiness probe failed: dial tcp 10.132.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:26:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:15Z" name: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx.18a8b407548625d6 namespace: kserve-ci-e2e-test resourceVersion: "15512" uid: 978b92ce-e47a-4219-9308-2c22be2ae509 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: b20ca612-2287-4a0f-a3fc-210e26f59ada kind: Event lastTimestamp: "2026-04-22T14:28:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:11Z" name: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx.18a8b41d6bbe8823 namespace: kserve-ci-e2e-test resourceVersion: "15390" uid: f2f06949-4932-4641-80ff-06687f448f6e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-dc953-predictor-559867c7 namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: ee2f782d-48af-4f54-8966-ce43bc9b9239 kind: Event lastTimestamp: "2026-04-22T14:26:31Z" message: 'Created pod: isvc-sklearn-graph-raw-dc953-predictor-559867c7-vt6bx' metadata: creationTimestamp: "2026-04-22T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:31Z" name: isvc-sklearn-graph-raw-dc953-predictor-559867c7.18a8b4060ffc9049 namespace: kserve-ci-e2e-test resourceVersion: "14454" uid: 961c437c-da2a-46af-9312-436df025fb11 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-dc953-predictor namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: 76100ef5-6e8b-4a85-94f0-697e04fa0c63 kind: Event lastTimestamp: "2026-04-22T14:26:31Z" message: Scaled up replica set isvc-sklearn-graph-raw-dc953-predictor-559867c7 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:31Z" name: isvc-sklearn-graph-raw-dc953-predictor.18a8b4060b74ef81 namespace: kserve-ci-e2e-test resourceVersion: "14445" uid: 3e7c1643-f125-493e-84cb-4de9defa0765 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:26:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-dc953-predictor namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 99ffc435-7e94-42df-ab6d-2499a0a2d5c6 kind: Event lastTimestamp: "2026-04-22T14:27:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:26:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:01Z" name: isvc-sklearn-graph-raw-dc953-predictor.18a8b4098a7e9669 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 43ada84d-ad79-46da-9f49-223353162ad9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:26:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-dc953-predictor namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 99ffc435-7e94-42df-ab6d-2499a0a2d5c6 kind: Event lastTimestamp: "2026-04-22T14:27:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:26:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:01Z" name: isvc-sklearn-graph-raw-dc953-predictor.18a8b4098ae1d46b namespace: kserve-ci-e2e-test resourceVersion: "14773" uid: 4e76b077-1631-46f0-8b7c-8e86d00f1ba2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T14:27:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-dc953-predictor namespace: kserve-ci-e2e-test resourceVersion: "14644" uid: 99ffc435-7e94-42df-ab6d-2499a0a2d5c6 kind: Event lastTimestamp: "2026-04-22T14:28:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:27:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:28:01Z" name: isvc-sklearn-graph-raw-dc953-predictor.18a8b41088e197ba namespace: kserve-ci-e2e-test resourceVersion: "15236" uid: 5b0bee63-8b22-476f-8627-ccae404cd2c9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T14:27:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-dc953-predictor namespace: kserve-ci-e2e-test resourceVersion: "14644" uid: 99ffc435-7e94-42df-ab6d-2499a0a2d5c6 kind: Event lastTimestamp: "2026-04-22T14:28:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:27:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:28:01Z" name: isvc-sklearn-graph-raw-dc953-predictor.18a8b410894cf23f namespace: kserve-ci-e2e-test resourceVersion: "15239" uid: a2b36500-05e2-4685-ad2a-23bd348909a6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-22T14:26:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-dc953 namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: 7b1a5fb2-34cb-4e10-938c-1954658244cf kind: Event lastTimestamp: "2026-04-22T14:27:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:27:45Z" name: isvc-sklearn-graph-raw-dc953.18a8b4060b77cdf6 namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: c1f34351-e3af-4108-9558-9ddc567fcc5d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-dc953 namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: 7b1a5fb2-34cb-4e10-938c-1954658244cf kind: Event lastTimestamp: "2026-04-22T14:26:31Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-dc953": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-dc953": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:26:31Z" name: isvc-sklearn-graph-raw-dc953.18a8b4060c22c0cf namespace: kserve-ci-e2e-test resourceVersion: "14447" uid: 442d34e7-617e-4ca8-8e06-2d805736f31b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:27:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-dc953 namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: 7b1a5fb2-34cb-4e10-938c-1954658244cf kind: Event lastTimestamp: "2026-04-22T14:27:45Z" message: InferenceService [isvc-sklearn-graph-raw-dc953] is Ready metadata: creationTimestamp: "2026-04-22T14:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:27:45Z" name: isvc-sklearn-graph-raw-dc953.18a8b4176cf59c6d namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 66a918a6-f661-4556-85b4-d43883feabdc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:28:11.484709Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8 namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: 60ab39b6-b02c-444b-86e9-400d6ac96d03 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8 to ip-10-0-129-161.ec2.internal metadata: creationTimestamp: "2026-04-22T14:28:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:28:11Z" name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8.18a8b41d6dac399c namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: 2d340a10-42bd-4e4e-a43e-bef1927a185f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56d6f9bdf8-db99f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8 namespace: kserve-ci-e2e-test resourceVersion: "15415" uid: 60ab39b6-b02c-444b-86e9-400d6ac96d03 kind: Event lastTimestamp: "2026-04-22T14:28:11Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:28:11Z" name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8.18a8b41d86b3df10 namespace: kserve-ci-e2e-test resourceVersion: "15459" uid: 965f9656-5ddb-48be-bc01-9253331a2be0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 60ab39b6-b02c-444b-86e9-400d6ac96d03 kind: Event lastTimestamp: "2026-04-22T14:28:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:498563386b50b61380307187059de966bc0a117e7161dd79d36731f0b2ebd43d" already present on machine metadata: creationTimestamp: "2026-04-22T14:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:11Z" name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8.18a8b41d881cbbcb namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 6e94d832-f58b-458c-bd21-cccd493f2e76 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 60ab39b6-b02c-444b-86e9-400d6ac96d03 kind: Event lastTimestamp: "2026-04-22T14:28:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T14:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:11Z" name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8.18a8b41d8afdefdb namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: baeac8a4-8424-4bdb-8314-0f1aa45ba958 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 60ab39b6-b02c-444b-86e9-400d6ac96d03 kind: Event lastTimestamp: "2026-04-22T14:28:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T14:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:11Z" name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8.18a8b41d8be2afdc namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: f3be26d8-f29e-4648-8dfe-279b43b9ea3d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 60ab39b6-b02c-444b-86e9-400d6ac96d03 kind: Event lastTimestamp: "2026-04-22T14:28:15Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:15Z" name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8.18a8b41e71ed20a0 namespace: kserve-ci-e2e-test resourceVersion: "15517" uid: 64344ae1-5fef-4da3-8c99-5d007cdaed3f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 60ab39b6-b02c-444b-86e9-400d6ac96d03 kind: Event lastTimestamp: "2026-04-22T14:28:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:15Z" name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8.18a8b41e756119eb namespace: kserve-ci-e2e-test resourceVersion: "15525" uid: 0a6d90d4-8978-4a53-b6dd-5405d8383e90 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 60ab39b6-b02c-444b-86e9-400d6ac96d03 kind: Event lastTimestamp: "2026-04-22T14:28:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:15Z" name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8.18a8b41e75dd5f0e namespace: kserve-ci-e2e-test resourceVersion: "15527" uid: d96123e9-39cd-4c51-a3da-76c2a081db83 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-22T14:28:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 60ab39b6-b02c-444b-86e9-400d6ac96d03 kind: Event lastTimestamp: "2026-04-22T14:29:17Z" message: 'Readiness probe failed: dial tcp 10.132.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:29:17Z" name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8.18a8b41eae275b99 namespace: kserve-ci-e2e-test resourceVersion: "15998" uid: 1c0bb916-3c27-4daa-8e2b-e45933594ec4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 60ab39b6-b02c-444b-86e9-400d6ac96d03 kind: Event lastTimestamp: "2026-04-22T14:29:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:29:51Z" name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8.18a8b434c3a47222 namespace: kserve-ci-e2e-test resourceVersion: "16343" uid: 9986edf9-4296-49a9-b210-ac340d2584cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465 namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: b6fd31fb-b335-43c7-966c-8a036cdae6c9 kind: Event lastTimestamp: "2026-04-22T14:28:11Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465-snqg8' metadata: creationTimestamp: "2026-04-22T14:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:28:11Z" name: isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465.18a8b41d6d13753a namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: 0d2c691a-985d-4151-96c5-fe75045cc585 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-2ea51-predictor namespace: kserve-ci-e2e-test resourceVersion: "15364" uid: 37a2cca7-fbca-43fd-a875-7687f6e33ba5 kind: Event lastTimestamp: "2026-04-22T14:28:11Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-2ea51-predictor-784c998465 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:28:11Z" name: isvc-sklearn-graph-raw-hpa-2ea51-predictor.18a8b41d67dc0c2c namespace: kserve-ci-e2e-test resourceVersion: "15372" uid: d2b9717b-3c23-4df7-9a4f-fbedd9b0f634 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:28:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-2ea51-predictor namespace: kserve-ci-e2e-test resourceVersion: "15368" uid: 80dd36b1-9564-4d3b-931c-2bb83345d8bb kind: Event lastTimestamp: "2026-04-22T14:28:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:28:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:28:56Z" name: isvc-sklearn-graph-raw-hpa-2ea51-predictor.18a8b420e67a63ff namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: b8a77c1c-c1c8-460c-9092-f5be0d21fe65 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:28:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-2ea51-predictor namespace: kserve-ci-e2e-test resourceVersion: "15368" uid: 80dd36b1-9564-4d3b-931c-2bb83345d8bb kind: Event lastTimestamp: "2026-04-22T14:28:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:28:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:28:56Z" name: isvc-sklearn-graph-raw-hpa-2ea51-predictor.18a8b420e6f5fce4 namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: f366db3b-55a7-4126-afb7-a01f6d9b62e6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:29:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-2ea51-predictor namespace: kserve-ci-e2e-test resourceVersion: "15614" uid: 80dd36b1-9564-4d3b-931c-2bb83345d8bb kind: Event lastTimestamp: "2026-04-22T14:29:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:29:41Z" name: isvc-sklearn-graph-raw-hpa-2ea51-predictor.18a8b42b6401d36a namespace: kserve-ci-e2e-test resourceVersion: "16196" uid: 46574e5c-fbfc-4f68-b6a2-8b9f90d15df0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:29:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-2ea51-predictor namespace: kserve-ci-e2e-test resourceVersion: "15614" uid: 80dd36b1-9564-4d3b-931c-2bb83345d8bb kind: Event lastTimestamp: "2026-04-22T14:29:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:29:41Z" name: isvc-sklearn-graph-raw-hpa-2ea51-predictor.18a8b42b6473a575 namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: ca6faafb-f9b2-4436-a31d-d79c1a22a02e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-22T14:28:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-2ea51 namespace: kserve-ci-e2e-test resourceVersion: "15359" uid: e9dd41fe-85bd-4db3-853f-fd33ca5da2a4 kind: Event lastTimestamp: "2026-04-22T14:29:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:29:28Z" name: isvc-sklearn-graph-raw-hpa-2ea51.18a8b41d67d0d86e namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: ad48aeab-4f92-464a-815a-60e49614d3a0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-2ea51 namespace: kserve-ci-e2e-test resourceVersion: "16085" uid: e9dd41fe-85bd-4db3-853f-fd33ca5da2a4 kind: Event lastTimestamp: "2026-04-22T14:29:27Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-2ea51] is Ready metadata: creationTimestamp: "2026-04-22T14:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:29:27Z" name: isvc-sklearn-graph-raw-hpa-2ea51.18a8b42f3d066d1a namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 52f3999a-1f12-45e1-9655-ad93ecee0611 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:31:27.006873Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w namespace: kserve-ci-e2e-test resourceVersion: "17219" uid: 0430015c-707f-4728-9170-ed9626aaad1b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w to ip-10-0-129-161.ec2.internal metadata: creationTimestamp: "2026-04-22T14:31:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:31:27Z" name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w.18a8b44af3b3b550 namespace: kserve-ci-e2e-test resourceVersion: "17225" uid: a8f1c8c6-2ef0-49cd-831b-671cad417d43 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56d6f9bdf8-db99f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:31:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w namespace: kserve-ci-e2e-test resourceVersion: "17229" uid: 0430015c-707f-4728-9170-ed9626aaad1b kind: Event lastTimestamp: "2026-04-22T14:31:27Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:31:27Z" name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w.18a8b44b0cb6a447 namespace: kserve-ci-e2e-test resourceVersion: "17252" uid: 9141b822-64cf-4c74-b412-9ea2aa463397 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w namespace: kserve-ci-e2e-test resourceVersion: "17223" uid: 0430015c-707f-4728-9170-ed9626aaad1b kind: Event lastTimestamp: "2026-04-22T14:31:27Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:498563386b50b61380307187059de966bc0a117e7161dd79d36731f0b2ebd43d" already present on machine metadata: creationTimestamp: "2026-04-22T14:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:31:27Z" name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w.18a8b44b0deddac1 namespace: kserve-ci-e2e-test resourceVersion: "17254" uid: a1d1a16c-1b92-4523-a1e5-a3c8c912ff3b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w namespace: kserve-ci-e2e-test resourceVersion: "17223" uid: 0430015c-707f-4728-9170-ed9626aaad1b kind: Event lastTimestamp: "2026-04-22T14:31:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T14:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:31:27Z" name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w.18a8b44b10c7942d namespace: kserve-ci-e2e-test resourceVersion: "17257" uid: 408e1736-7a7d-4e9d-8554-20700ee19209 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w namespace: kserve-ci-e2e-test resourceVersion: "17223" uid: 0430015c-707f-4728-9170-ed9626aaad1b kind: Event lastTimestamp: "2026-04-22T14:31:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T14:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:31:27Z" name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w.18a8b44b1134a6ef namespace: kserve-ci-e2e-test resourceVersion: "17258" uid: 8508a9cd-7e04-49e7-bf0a-955c0f7fdd48 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:31:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w namespace: kserve-ci-e2e-test resourceVersion: "17223" uid: 0430015c-707f-4728-9170-ed9626aaad1b kind: Event lastTimestamp: "2026-04-22T14:31:32Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:31:32Z" name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w.18a8b44c363370aa namespace: kserve-ci-e2e-test resourceVersion: "17296" uid: 8edc807f-24a9-4218-b16f-d7ff3ae51ddc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:31:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w namespace: kserve-ci-e2e-test resourceVersion: "17223" uid: 0430015c-707f-4728-9170-ed9626aaad1b kind: Event lastTimestamp: "2026-04-22T14:31:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:31:32Z" name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w.18a8b44c38e593fe namespace: kserve-ci-e2e-test resourceVersion: "17300" uid: 24cbac77-9b59-486e-9926-1b4fc5563625 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:31:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w namespace: kserve-ci-e2e-test resourceVersion: "17223" uid: 0430015c-707f-4728-9170-ed9626aaad1b kind: Event lastTimestamp: "2026-04-22T14:31:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:31:32Z" name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w.18a8b44c3954cfcd namespace: kserve-ci-e2e-test resourceVersion: "17301" uid: bec7e8de-eace-4ffa-802e-c31a4a51d67f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 16 eventTime: null firstTimestamp: "2026-04-22T14:31:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w namespace: kserve-ci-e2e-test resourceVersion: "17223" uid: 0430015c-707f-4728-9170-ed9626aaad1b kind: Event lastTimestamp: "2026-04-22T14:33:59Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:31:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:33:59Z" name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w.18a8b44c72202bb0 namespace: kserve-ci-e2e-test resourceVersion: "18422" uid: edf57d1e-a876-442e-a9d1-12f37e175ef7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:33:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w namespace: kserve-ci-e2e-test resourceVersion: "17223" uid: 0430015c-707f-4728-9170-ed9626aaad1b kind: Event lastTimestamp: "2026-04-22T14:33:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:33:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:33:57Z" name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w.18a8b46dec5c7b91 namespace: kserve-ci-e2e-test resourceVersion: "18369" uid: 3c8c5279-5a31-4564-8a4f-c55f2808f74d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:31:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: 651b0f59-bd22-49bd-807d-3c60d5143842 kind: Event lastTimestamp: "2026-04-22T14:31:26Z" message: 'Created pod: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c-cgc8w' metadata: creationTimestamp: "2026-04-22T14:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:31:26Z" name: isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c.18a8b44af329aaa2 namespace: kserve-ci-e2e-test resourceVersion: "17222" uid: a9727435-c19d-4910-bb5d-5b5599294d6d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:31:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-0ba4e-predictor namespace: kserve-ci-e2e-test resourceVersion: "17195" uid: 7045522a-9ace-4b41-a3e2-5c1200de0efb kind: Event lastTimestamp: "2026-04-22T14:31:26Z" message: Scaled up replica set isvc-sklearn-scale-raw-0ba4e-predictor-69458d988c from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:31:26Z" name: isvc-sklearn-scale-raw-0ba4e-predictor.18a8b44af0c51fb4 namespace: kserve-ci-e2e-test resourceVersion: "17207" uid: 5fd3e8a0-fee5-4cac-b3d6-cd13ca20268c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T14:31:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-0ba4e-predictor namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 3ff9c57e-dde2-4d15-9439-ce5a6721d585 kind: Event lastTimestamp: "2026-04-22T14:32:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:32:27Z" name: isvc-sklearn-scale-raw-0ba4e-predictor.18a8b44e6f96574b namespace: kserve-ci-e2e-test resourceVersion: "17699" uid: 5b3143cc-953d-4ea6-90b5-4aab7b47520d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T14:31:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-0ba4e-predictor namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 3ff9c57e-dde2-4d15-9439-ce5a6721d585 kind: Event lastTimestamp: "2026-04-22T14:32:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:32:27Z" name: isvc-sklearn-scale-raw-0ba4e-predictor.18a8b44e6ff890d8 namespace: kserve-ci-e2e-test resourceVersion: "17700" uid: f6bda010-ba80-41ae-8631-6af8e00985f6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:32:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-0ba4e-predictor namespace: kserve-ci-e2e-test resourceVersion: "17375" uid: 3ff9c57e-dde2-4d15-9439-ce5a6721d585 kind: Event lastTimestamp: "2026-04-22T14:33:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:33:57Z" name: isvc-sklearn-scale-raw-0ba4e-predictor.18a8b45c6b7d1be1 namespace: kserve-ci-e2e-test resourceVersion: "18339" uid: c7900933-b84f-4e97-8a0a-90335e3662cd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:32:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-0ba4e-predictor namespace: kserve-ci-e2e-test resourceVersion: "17375" uid: 3ff9c57e-dde2-4d15-9439-ce5a6721d585 kind: Event lastTimestamp: "2026-04-22T14:33:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:33:57Z" name: isvc-sklearn-scale-raw-0ba4e-predictor.18a8b45c6be2ba53 namespace: kserve-ci-e2e-test resourceVersion: "18341" uid: ee8c3f10-9aec-4cc2-9c84-c2e615e5ade1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-22T14:31:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-0ba4e namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: d19a041f-4c0e-4bb0-9998-3d7496217df9 kind: Event lastTimestamp: "2026-04-22T14:33:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:33:49Z" name: isvc-sklearn-scale-raw-0ba4e.18a8b44af0a3ba90 namespace: kserve-ci-e2e-test resourceVersion: "18284" uid: 58cc7bf5-e9a5-4fb9-9bcc-855f70e6ae97 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:31:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-0ba4e namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: d19a041f-4c0e-4bb0-9998-3d7496217df9 kind: Event lastTimestamp: "2026-04-22T14:31:26Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-0ba4e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-0ba4e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:31:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:31:26Z" name: isvc-sklearn-scale-raw-0ba4e.18a8b44af0f0acc2 namespace: kserve-ci-e2e-test resourceVersion: "17210" uid: b599a228-e6f1-4f58-a76e-fcf9e467671d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:33:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-0ba4e namespace: kserve-ci-e2e-test resourceVersion: "18282" uid: d19a041f-4c0e-4bb0-9998-3d7496217df9 kind: Event lastTimestamp: "2026-04-22T14:33:49Z" message: InferenceService [isvc-sklearn-scale-raw-0ba4e] is Ready metadata: creationTimestamp: "2026-04-22T14:33:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:33:49Z" name: isvc-sklearn-scale-raw-0ba4e.18a8b46c2cb3412e namespace: kserve-ci-e2e-test resourceVersion: "18283" uid: c7c9dc39-786d-4ecc-a275-59abfb596264 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:26:31.239317Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4 namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: b4732b54-3b30-4a6a-9533-205f36c1d827 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4 to ip-10-0-129-161.ec2.internal metadata: creationTimestamp: "2026-04-22T14:26:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:26:31Z" name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4.18a8b4061694feb2 namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: 8128c215-c2a6-44fc-9979-680d3014b7e6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56d6f9bdf8-db99f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4 namespace: kserve-ci-e2e-test resourceVersion: "14493" uid: b4732b54-3b30-4a6a-9533-205f36c1d827 kind: Event lastTimestamp: "2026-04-22T14:26:31Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:26:31Z" name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4.18a8b4062fa9b068 namespace: kserve-ci-e2e-test resourceVersion: "14509" uid: 74d767e9-8a5a-4039-b2e4-216ce8b797e2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4 namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b4732b54-3b30-4a6a-9533-205f36c1d827 kind: Event lastTimestamp: "2026-04-22T14:26:31Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:498563386b50b61380307187059de966bc0a117e7161dd79d36731f0b2ebd43d" already present on machine metadata: creationTimestamp: "2026-04-22T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:31Z" name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4.18a8b40630dbf910 namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: 74bb0ef9-9935-4a5d-856e-2b090887c674 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4 namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b4732b54-3b30-4a6a-9533-205f36c1d827 kind: Event lastTimestamp: "2026-04-22T14:26:31Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:31Z" name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4.18a8b4063328a604 namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 9c6eca70-0706-4068-a3c3-3157877bc096 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4 namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b4732b54-3b30-4a6a-9533-205f36c1d827 kind: Event lastTimestamp: "2026-04-22T14:26:31Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:31Z" name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4.18a8b406338961e5 namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 403570fb-a0fc-4e18-95fa-4c951ebb89dc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4 namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b4732b54-3b30-4a6a-9533-205f36c1d827 kind: Event lastTimestamp: "2026-04-22T14:26:35Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-22T14:26:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:35Z" name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4.18a8b40718793a3b namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: a31af698-4cd7-4cac-9a18-b140efde3319 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4 namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b4732b54-3b30-4a6a-9533-205f36c1d827 kind: Event lastTimestamp: "2026-04-22T14:26:57Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 21.688s (21.689s including waiting). Image size: 1306326779 bytes.' metadata: creationTimestamp: "2026-04-22T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:57Z" name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4.18a8b40c253d95d0 namespace: kserve-ci-e2e-test resourceVersion: "14733" uid: e0fada97-3e3b-4a9f-9b31-4a33d6937abc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4 namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b4732b54-3b30-4a6a-9533-205f36c1d827 kind: Event lastTimestamp: "2026-04-22T14:26:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:57Z" name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4.18a8b40c28483724 namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: 0d1f114f-20ee-4f99-925f-028e1c8ecbb6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4 namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b4732b54-3b30-4a6a-9533-205f36c1d827 kind: Event lastTimestamp: "2026-04-22T14:26:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:26:57Z" name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4.18a8b40c28cc0920 namespace: kserve-ci-e2e-test resourceVersion: "14735" uid: 6f0b8a6c-7d07-4fdd-837e-90444062f228 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T14:26:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4 namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b4732b54-3b30-4a6a-9533-205f36c1d827 kind: Event lastTimestamp: "2026-04-22T14:27:48Z" message: 'Readiness probe failed: dial tcp 10.132.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:26:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:27:48Z" name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4.18a8b40c3bf8c855 namespace: kserve-ci-e2e-test resourceVersion: "15127" uid: 1b30111b-e49c-411f-b03a-8453aa7db1da reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4 namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: b4732b54-3b30-4a6a-9533-205f36c1d827 kind: Event lastTimestamp: "2026-04-22T14:28:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:11Z" name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4.18a8b41d76005ced namespace: kserve-ci-e2e-test resourceVersion: "15451" uid: a2e06cbc-c833-4c70-93cc-ce9fc2660efe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59 namespace: kserve-ci-e2e-test resourceVersion: "14471" uid: 29b15ff3-2e2b-4063-a5d6-09fdb4aed769 kind: Event lastTimestamp: "2026-04-22T14:26:31Z" message: 'Created pod: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59-rxlq4' metadata: creationTimestamp: "2026-04-22T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:31Z" name: isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59.18a8b40615d13b30 namespace: kserve-ci-e2e-test resourceVersion: "14481" uid: 59ef067d-c067-4040-b5e9-78cbabe270ee reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:26:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-dc953-predictor namespace: kserve-ci-e2e-test resourceVersion: "14469" uid: 991e2e02-3153-4f08-80da-aed2e967e293 kind: Event lastTimestamp: "2026-04-22T14:26:31Z" message: Scaled up replica set isvc-xgboost-graph-raw-dc953-predictor-b86bc8b59 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:26:31Z" name: isvc-xgboost-graph-raw-dc953-predictor.18a8b4061332756f namespace: kserve-ci-e2e-test resourceVersion: "14476" uid: 68daca5b-336b-4c43-8c07-d3fc00a791fc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T14:26:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-dc953-predictor namespace: kserve-ci-e2e-test resourceVersion: "14470" uid: 282c1e2d-78a7-4323-a681-5361f201f892 kind: Event lastTimestamp: "2026-04-22T14:27:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:26:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:31Z" name: isvc-xgboost-graph-raw-dc953-predictor.18a8b4099173674e namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 98841cdc-1da6-4db2-b29a-3add833c7c5c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T14:26:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-dc953-predictor namespace: kserve-ci-e2e-test resourceVersion: "14470" uid: 282c1e2d-78a7-4323-a681-5361f201f892 kind: Event lastTimestamp: "2026-04-22T14:27:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:26:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:27:31Z" name: isvc-xgboost-graph-raw-dc953-predictor.18a8b40991dd5fbd namespace: kserve-ci-e2e-test resourceVersion: "14994" uid: 09922954-0284-4306-8c46-8af3f5c76097 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:27:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-dc953-predictor namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 282c1e2d-78a7-4323-a681-5361f201f892 kind: Event lastTimestamp: "2026-04-22T14:28:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:28:01Z" name: isvc-xgboost-graph-raw-dc953-predictor.18a8b4178caea72a namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: faf40764-cdd7-41f2-a7ec-bd43b9e509db reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T14:27:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-dc953-predictor namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 282c1e2d-78a7-4323-a681-5361f201f892 kind: Event lastTimestamp: "2026-04-22T14:28:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:28:01Z" name: isvc-xgboost-graph-raw-dc953-predictor.18a8b4178d177dbd namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: 59658d1e-8034-4e64-995f-a28f352725e3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T14:26:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-dc953 namespace: kserve-ci-e2e-test resourceVersion: "14452" uid: c07e0e2e-72cf-49ee-ad8a-230bc4c3228e kind: Event lastTimestamp: "2026-04-22T14:27:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:27:58Z" name: isvc-xgboost-graph-raw-dc953.18a8b40612dbe9ba namespace: kserve-ci-e2e-test resourceVersion: "15203" uid: a6e6c0fe-957a-474c-9504-4b612240bcbe reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:27:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-dc953 namespace: kserve-ci-e2e-test resourceVersion: "15201" uid: c07e0e2e-72cf-49ee-ad8a-230bc4c3228e kind: Event lastTimestamp: "2026-04-22T14:27:58Z" message: InferenceService [isvc-xgboost-graph-raw-dc953] is Ready metadata: creationTimestamp: "2026-04-22T14:27:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:27:58Z" name: isvc-xgboost-graph-raw-dc953.18a8b41a7475fb8e namespace: kserve-ci-e2e-test resourceVersion: "15202" uid: 5def14f0-d80a-4c7f-81b1-7871e14730ba reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:28:11.565808Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9 namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: 4ae5343d-0bc9-4a20-939c-3e96db7446ed kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9 to ip-10-0-129-161.ec2.internal metadata: creationTimestamp: "2026-04-22T14:28:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:28:11Z" name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9.18a8b41d7281b252 namespace: kserve-ci-e2e-test resourceVersion: "15425" uid: 924fb47c-d418-4594-9c26-facd45ae5faa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56d6f9bdf8-db99f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9 namespace: kserve-ci-e2e-test resourceVersion: "15434" uid: 4ae5343d-0bc9-4a20-939c-3e96db7446ed kind: Event lastTimestamp: "2026-04-22T14:28:11Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:28:11Z" name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9.18a8b41d8bc0d865 namespace: kserve-ci-e2e-test resourceVersion: "15466" uid: 404822a1-8450-4144-bfa7-c08097ce9c5f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9 namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 4ae5343d-0bc9-4a20-939c-3e96db7446ed kind: Event lastTimestamp: "2026-04-22T14:28:12Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:498563386b50b61380307187059de966bc0a117e7161dd79d36731f0b2ebd43d" already present on machine metadata: creationTimestamp: "2026-04-22T14:28:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:12Z" name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9.18a8b41d8db1f371 namespace: kserve-ci-e2e-test resourceVersion: "15469" uid: 2a56554e-3291-43be-bb56-897ec163e949 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9 namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 4ae5343d-0bc9-4a20-939c-3e96db7446ed kind: Event lastTimestamp: "2026-04-22T14:28:12Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T14:28:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:12Z" name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9.18a8b41d90492adf namespace: kserve-ci-e2e-test resourceVersion: "15470" uid: 5205ab31-e2c7-42c0-a51f-ffb14add4863 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9 namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 4ae5343d-0bc9-4a20-939c-3e96db7446ed kind: Event lastTimestamp: "2026-04-22T14:28:12Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T14:28:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:12Z" name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9.18a8b41d90bddb8a namespace: kserve-ci-e2e-test resourceVersion: "15471" uid: 3ad202fc-c6d3-4860-8f08-c6ba5978c29f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9 namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 4ae5343d-0bc9-4a20-939c-3e96db7446ed kind: Event lastTimestamp: "2026-04-22T14:28:15Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-22T14:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:15Z" name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9.18a8b41e7217ca10 namespace: kserve-ci-e2e-test resourceVersion: "15518" uid: 7a7b3925-8bd5-4a03-bc89-101e284903e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9 namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 4ae5343d-0bc9-4a20-939c-3e96db7446ed kind: Event lastTimestamp: "2026-04-22T14:28:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:15Z" name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9.18a8b41e7573a43a namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: 3830dddd-d2d6-4c7e-8b28-47bb8e29d654 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9 namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 4ae5343d-0bc9-4a20-939c-3e96db7446ed kind: Event lastTimestamp: "2026-04-22T14:28:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:15Z" name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9.18a8b41e75eb7d3d namespace: kserve-ci-e2e-test resourceVersion: "15530" uid: 6747bee8-7658-4783-a508-ac8b2db01814 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T14:28:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9 namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 4ae5343d-0bc9-4a20-939c-3e96db7446ed kind: Event lastTimestamp: "2026-04-22T14:29:07Z" message: 'Readiness probe failed: dial tcp 10.132.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:29:07Z" name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9.18a8b41eae1227de namespace: kserve-ci-e2e-test resourceVersion: "15924" uid: 124a58b9-9e89-436c-bf9d-fc1849b787fb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9 namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 4ae5343d-0bc9-4a20-939c-3e96db7446ed kind: Event lastTimestamp: "2026-04-22T14:29:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:29:51Z" name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9.18a8b434cb9ca0de namespace: kserve-ci-e2e-test resourceVersion: "16378" uid: e77fadb9-4e9c-4cb8-beb8-d2ede30b74ba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5 namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 6995cb8d-d861-442e-bb82-e7e111c88657 kind: Event lastTimestamp: "2026-04-22T14:28:11Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5-n8nc9' metadata: creationTimestamp: "2026-04-22T14:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:28:11Z" name: isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5.18a8b41d71acc18b namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 59f1ea71-fd03-4037-a5e2-e8593e080953 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-2ea51-predictor namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: d6233054-7455-4873-8376-8febe9b00274 kind: Event lastTimestamp: "2026-04-22T14:28:11Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-2ea51-predictor-5d9d79f9c5 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:28:11Z" name: isvc-xgboost-graph-raw-hpa-2ea51-predictor.18a8b41d6ebcdb6b namespace: kserve-ci-e2e-test resourceVersion: "15413" uid: 318b21b1-028e-4a0a-9865-8c907bdf3a34 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:28:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-2ea51-predictor namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: d2ddf3bb-6ffc-49e6-b33a-5d4f0e0fab1f kind: Event lastTimestamp: "2026-04-22T14:28:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:28:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:28:56Z" name: isvc-xgboost-graph-raw-hpa-2ea51-predictor.18a8b420ec9c4234 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 823ae6a0-95f6-426f-bbc1-8aa5632c323d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:28:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-2ea51-predictor namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: d2ddf3bb-6ffc-49e6-b33a-5d4f0e0fab1f kind: Event lastTimestamp: "2026-04-22T14:28:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:28:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:28:56Z" name: isvc-xgboost-graph-raw-hpa-2ea51-predictor.18a8b420ed24fc02 namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: f135f77b-7c03-421f-915a-82c9842b222c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:29:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-2ea51-predictor namespace: kserve-ci-e2e-test resourceVersion: "15618" uid: d2ddf3bb-6ffc-49e6-b33a-5d4f0e0fab1f kind: Event lastTimestamp: "2026-04-22T14:29:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:29:41Z" name: isvc-xgboost-graph-raw-hpa-2ea51-predictor.18a8b42b69aa4b08 namespace: kserve-ci-e2e-test resourceVersion: "16213" uid: 0f8c0659-5f7d-49a0-ac0b-ff60116d05a0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T14:29:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-2ea51-predictor namespace: kserve-ci-e2e-test resourceVersion: "15618" uid: d2ddf3bb-6ffc-49e6-b33a-5d4f0e0fab1f kind: Event lastTimestamp: "2026-04-22T14:29:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:29:41Z" name: isvc-xgboost-graph-raw-hpa-2ea51-predictor.18a8b42b6a153217 namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: 0c316e81-3161-4f7d-b740-74f1c8a08ddd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T14:28:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-2ea51 namespace: kserve-ci-e2e-test resourceVersion: "15388" uid: 48a057d4-3d86-4ee5-93bb-b72680a8b715 kind: Event lastTimestamp: "2026-04-22T14:29:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:29:17Z" name: isvc-xgboost-graph-raw-hpa-2ea51.18a8b41d6ded8362 namespace: kserve-ci-e2e-test resourceVersion: "16008" uid: 9a901d75-09d7-4e72-abbd-d75f3034a2df reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-2ea51 namespace: kserve-ci-e2e-test resourceVersion: "15411" uid: 48a057d4-3d86-4ee5-93bb-b72680a8b715 kind: Event lastTimestamp: "2026-04-22T14:28:11Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-2ea51": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-2ea51": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:28:11Z" name: isvc-xgboost-graph-raw-hpa-2ea51.18a8b41d746603b6 namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: 5cc2e7ad-8f97-4615-82f9-b05f27ce3ca6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-2ea51 namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: 48a057d4-3d86-4ee5-93bb-b72680a8b715 kind: Event lastTimestamp: "2026-04-22T14:29:17Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-2ea51] is Ready metadata: creationTimestamp: "2026-04-22T14:29:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:29:17Z" name: isvc-xgboost-graph-raw-hpa-2ea51.18a8b42ce6d4a4c9 namespace: kserve-ci-e2e-test resourceVersion: "16007" uid: 4b0b05f4-8373-4e17-bcbd-4f30b9a3dc05 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:29:51.747187Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-8d0fa-predictor-5f649c8785-gh8cl namespace: kserve-ci-e2e-test resourceVersion: "16346" uid: cff0fcbf-3994-4c6d-b36d-4520955398b2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-8d0fa-predictor-5f649c8785-gh8cl to ip-10-0-129-161.ec2.internal metadata: creationTimestamp: "2026-04-22T14:29:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:29:51Z" name: message-dumper-raw-8d0fa-predictor-5f649c8785-gh8cl.18a8b434c5c83885 namespace: kserve-ci-e2e-test resourceVersion: "16353" uid: 2db0c7b5-26e0-48b7-bbb4-1f663a8d4c00 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56d6f9bdf8-db99f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:51Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-8d0fa-predictor-5f649c8785-gh8cl namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: cff0fcbf-3994-4c6d-b36d-4520955398b2 kind: Event lastTimestamp: "2026-04-22T14:29:51Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:29:51Z" name: message-dumper-raw-8d0fa-predictor-5f649c8785-gh8cl.18a8b434ccd1b54c namespace: kserve-ci-e2e-test resourceVersion: "16379" uid: 6d33ea73-546e-4d5d-9804-0fad3de9162d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-8d0fa-predictor-5f649c8785-gh8cl namespace: kserve-ci-e2e-test resourceVersion: "16349" uid: cff0fcbf-3994-4c6d-b36d-4520955398b2 kind: Event lastTimestamp: "2026-04-22T14:29:51Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-22T14:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:29:51Z" name: message-dumper-raw-8d0fa-predictor-5f649c8785-gh8cl.18a8b434ce1c44e2 namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: e3434e75-6e9f-47ff-95b1-b5c21e8e5539 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-8d0fa-predictor-5f649c8785-gh8cl namespace: kserve-ci-e2e-test resourceVersion: "16349" uid: cff0fcbf-3994-4c6d-b36d-4520955398b2 kind: Event lastTimestamp: "2026-04-22T14:29:52Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.019s (1.019s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-22T14:29:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:29:52Z" name: message-dumper-raw-8d0fa-predictor-5f649c8785-gh8cl.18a8b4350adf81b3 namespace: kserve-ci-e2e-test resourceVersion: "16392" uid: 84a5b394-342a-4aa6-9e21-b7e2230e7499 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-8d0fa-predictor-5f649c8785-gh8cl namespace: kserve-ci-e2e-test resourceVersion: "16349" uid: cff0fcbf-3994-4c6d-b36d-4520955398b2 kind: Event lastTimestamp: "2026-04-22T14:29:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:29:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:29:52Z" name: message-dumper-raw-8d0fa-predictor-5f649c8785-gh8cl.18a8b4350dc6307a namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: cd966f8d-4e04-4c83-a4a6-4ebaca21ace8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-8d0fa-predictor-5f649c8785-gh8cl namespace: kserve-ci-e2e-test resourceVersion: "16349" uid: cff0fcbf-3994-4c6d-b36d-4520955398b2 kind: Event lastTimestamp: "2026-04-22T14:29:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:29:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:29:52Z" name: message-dumper-raw-8d0fa-predictor-5f649c8785-gh8cl.18a8b4350e2fcc06 namespace: kserve-ci-e2e-test resourceVersion: "16395" uid: 3b37a0bc-e813-4314-a79d-edda0f16043b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:31:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-8d0fa-predictor-5f649c8785-gh8cl namespace: kserve-ci-e2e-test resourceVersion: "16349" uid: cff0fcbf-3994-4c6d-b36d-4520955398b2 kind: Event lastTimestamp: "2026-04-22T14:31:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:31:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:31:27Z" name: message-dumper-raw-8d0fa-predictor-5f649c8785-gh8cl.18a8b44af759407e namespace: kserve-ci-e2e-test resourceVersion: "17245" uid: 55a43fc8-cb3f-49fd-96bf-e5da9c2b2a61 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-8d0fa-predictor-5f649c8785 namespace: kserve-ci-e2e-test resourceVersion: "16324" uid: d42784d0-af34-47dd-aaa3-067676c1ca2a kind: Event lastTimestamp: "2026-04-22T14:29:51Z" message: 'Created pod: message-dumper-raw-8d0fa-predictor-5f649c8785-gh8cl' metadata: creationTimestamp: "2026-04-22T14:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:29:51Z" name: message-dumper-raw-8d0fa-predictor-5f649c8785.18a8b434c5119b28 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: 72cda537-7039-4d1f-a078-4ec8df74a2ee reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-8d0fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: 63ab8334-6d6b-4f3a-aae3-25d320f9a947 kind: Event lastTimestamp: "2026-04-22T14:29:51Z" message: Scaled up replica set message-dumper-raw-8d0fa-predictor-5f649c8785 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:29:51Z" name: message-dumper-raw-8d0fa-predictor.18a8b434c15df549 namespace: kserve-ci-e2e-test resourceVersion: "16332" uid: b3e69334-0948-4177-bef6-fd63bc4a0c10 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:30:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-8d0fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "16323" uid: 9d33ea7f-e12e-4b3c-924b-de28e96298c3 kind: Event lastTimestamp: "2026-04-22T14:30:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:30:06Z" name: message-dumper-raw-8d0fa-predictor.18a8b4383ffbaea4 namespace: kserve-ci-e2e-test resourceVersion: "16570" uid: 6071494d-4b4f-4af5-ada4-cd6b5794f643 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:30:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-8d0fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "16323" uid: 9d33ea7f-e12e-4b3c-924b-de28e96298c3 kind: Event lastTimestamp: "2026-04-22T14:30:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:30:06Z" name: message-dumper-raw-8d0fa-predictor.18a8b4384069b93d namespace: kserve-ci-e2e-test resourceVersion: "16572" uid: 455f8f86-766f-4a8d-a631-77b9eca93725 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:30:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-8d0fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "16571" uid: 9d33ea7f-e12e-4b3c-924b-de28e96298c3 kind: Event lastTimestamp: "2026-04-22T14:30:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:30:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:30:21Z" name: message-dumper-raw-8d0fa-predictor.18a8b43bbf3a02cf namespace: kserve-ci-e2e-test resourceVersion: "16694" uid: 71b8c0d6-fa86-4140-92c5-366571778e96 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:30:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-8d0fa-predictor namespace: kserve-ci-e2e-test resourceVersion: "16571" uid: 9d33ea7f-e12e-4b3c-924b-de28e96298c3 kind: Event lastTimestamp: "2026-04-22T14:30:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:30:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:30:21Z" name: message-dumper-raw-8d0fa-predictor.18a8b43bbfb0242e namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: bfb429fb-0885-4ea0-8335-dce4e1c96d93 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-22T14:29:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-8d0fa namespace: kserve-ci-e2e-test resourceVersion: "16315" uid: ef60c0e2-4e8e-446a-89b4-060bfedb6476 kind: Event lastTimestamp: "2026-04-22T14:29:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:29:53Z" name: message-dumper-raw-8d0fa.18a8b434c126d130 namespace: kserve-ci-e2e-test resourceVersion: "16409" uid: c4451be5-9246-4d5f-9960-dafdea751025 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-8d0fa namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: ef60c0e2-4e8e-446a-89b4-060bfedb6476 kind: Event lastTimestamp: "2026-04-22T14:29:51Z" message: 'Failed to update status for InferenceService "message-dumper-raw-8d0fa": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-8d0fa": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:29:51Z" name: message-dumper-raw-8d0fa.18a8b434c4d44b4c namespace: kserve-ci-e2e-test resourceVersion: "16347" uid: bc44538c-7719-4f3a-ae1a-19fd1dc27006 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-8d0fa namespace: kserve-ci-e2e-test resourceVersion: "16407" uid: ef60c0e2-4e8e-446a-89b4-060bfedb6476 kind: Event lastTimestamp: "2026-04-22T14:29:53Z" message: InferenceService [message-dumper-raw-8d0fa] is Ready metadata: creationTimestamp: "2026-04-22T14:29:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:29:53Z" name: message-dumper-raw-8d0fa.18a8b4351d766780 namespace: kserve-ci-e2e-test resourceVersion: "16408" uid: e548df25-23db-4146-83db-1d433b6b3208 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:28:01.173682Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-dc953-5bffc7f67f-lqm6p namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: 2f2818ac-02c9-42a0-ac83-83dc7d6d80c9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-dc953-5bffc7f67f-lqm6p to ip-10-0-129-161.ec2.internal metadata: creationTimestamp: "2026-04-22T14:28:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:28:01Z" name: model-chainer-raw-dc953-5bffc7f67f-lqm6p.18a8b41b07167fcf namespace: kserve-ci-e2e-test resourceVersion: "15238" uid: 2f2fe2e7-f7f3-4073-af1b-db01c7461caa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56d6f9bdf8-db99f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:01Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-dc953-5bffc7f67f-lqm6p namespace: kserve-ci-e2e-test resourceVersion: "15232" uid: 2f2818ac-02c9-42a0-ac83-83dc7d6d80c9 kind: Event lastTimestamp: "2026-04-22T14:28:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-dc953-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T14:28:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:01Z" name: model-chainer-raw-dc953-5bffc7f67f-lqm6p.18a8b41b13716b28 namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: dc293e11-10de-4adf-827c-ddeb245690f2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:02Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-dc953-5bffc7f67f-lqm6p namespace: kserve-ci-e2e-test resourceVersion: "15242" uid: 2f2818ac-02c9-42a0-ac83-83dc7d6d80c9 kind: Event lastTimestamp: "2026-04-22T14:28:02Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:28:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:28:02Z" name: model-chainer-raw-dc953-5bffc7f67f-lqm6p.18a8b41b439c3123 namespace: kserve-ci-e2e-test resourceVersion: "15262" uid: da0e34da-4a9c-4ab8-bd2f-017881911c73 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-dc953} kind: Pod name: model-chainer-raw-dc953-5bffc7f67f-lqm6p namespace: kserve-ci-e2e-test resourceVersion: "15232" uid: 2f2818ac-02c9-42a0-ac83-83dc7d6d80c9 kind: Event lastTimestamp: "2026-04-22T14:28:02Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:5ddf7b68668839b048ac1f65cf22f5a7b21a1ddc89705b9e0ad240d64f017b66" metadata: creationTimestamp: "2026-04-22T14:28:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:02Z" name: model-chainer-raw-dc953-5bffc7f67f-lqm6p.18a8b41b44fdaa17 namespace: kserve-ci-e2e-test resourceVersion: "15264" uid: 2cb9d95a-4108-4d64-ad0a-07be57c1fe8c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-dc953} kind: Pod name: model-chainer-raw-dc953-5bffc7f67f-lqm6p namespace: kserve-ci-e2e-test resourceVersion: "15232" uid: 2f2818ac-02c9-42a0-ac83-83dc7d6d80c9 kind: Event lastTimestamp: "2026-04-22T14:28:04Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:5ddf7b68668839b048ac1f65cf22f5a7b21a1ddc89705b9e0ad240d64f017b66" in 2.264s (2.264s including waiting). Image size: 216233534 bytes.' metadata: creationTimestamp: "2026-04-22T14:28:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:04Z" name: model-chainer-raw-dc953-5bffc7f67f-lqm6p.18a8b41bcbf06fc3 namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: bbd8d9a9-a26d-4aa1-8196-882c1117634e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-dc953} kind: Pod name: model-chainer-raw-dc953-5bffc7f67f-lqm6p namespace: kserve-ci-e2e-test resourceVersion: "15232" uid: 2f2818ac-02c9-42a0-ac83-83dc7d6d80c9 kind: Event lastTimestamp: "2026-04-22T14:28:04Z" message: 'Created container: model-chainer-raw-dc953' metadata: creationTimestamp: "2026-04-22T14:28:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:04Z" name: model-chainer-raw-dc953-5bffc7f67f-lqm6p.18a8b41bcef4f7c7 namespace: kserve-ci-e2e-test resourceVersion: "15280" uid: 046b3845-950d-46d2-bdbb-1f1b64addc67 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-dc953} kind: Pod name: model-chainer-raw-dc953-5bffc7f67f-lqm6p namespace: kserve-ci-e2e-test resourceVersion: "15232" uid: 2f2818ac-02c9-42a0-ac83-83dc7d6d80c9 kind: Event lastTimestamp: "2026-04-22T14:28:04Z" message: Started container model-chainer-raw-dc953 metadata: creationTimestamp: "2026-04-22T14:28:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:04Z" name: model-chainer-raw-dc953-5bffc7f67f-lqm6p.18a8b41bcf6a942a namespace: kserve-ci-e2e-test resourceVersion: "15281" uid: 414a0834-758c-4050-be44-2e5c9e33c742 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-dc953} kind: Pod name: model-chainer-raw-dc953-5bffc7f67f-lqm6p namespace: kserve-ci-e2e-test resourceVersion: "15232" uid: 2f2818ac-02c9-42a0-ac83-83dc7d6d80c9 kind: Event lastTimestamp: "2026-04-22T14:28:11Z" message: Stopping container model-chainer-raw-dc953 metadata: creationTimestamp: "2026-04-22T14:28:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:11Z" name: model-chainer-raw-dc953-5bffc7f67f-lqm6p.18a8b41d61c6511d namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: f9d9b0f6-a727-4428-aa3d-80aa5688fe77 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:28:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-dc953} kind: Pod name: model-chainer-raw-dc953-5bffc7f67f-lqm6p namespace: kserve-ci-e2e-test resourceVersion: "15232" uid: 2f2818ac-02c9-42a0-ac83-83dc7d6d80c9 kind: Event lastTimestamp: "2026-04-22T14:28:40Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T14:28:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:28:40Z" name: model-chainer-raw-dc953-5bffc7f67f-lqm6p.18a8b41e706127ce namespace: kserve-ci-e2e-test resourceVersion: "15723" uid: 6d41ff90-a6e6-4244-b2e7-8040a7b07e08 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-dc953-5bffc7f67f namespace: kserve-ci-e2e-test resourceVersion: "15228" uid: 4a76fb68-06a1-48a0-818a-88ec8b60f124 kind: Event lastTimestamp: "2026-04-22T14:28:01Z" message: 'Created pod: model-chainer-raw-dc953-5bffc7f67f-lqm6p' metadata: creationTimestamp: "2026-04-22T14:28:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:28:01Z" name: model-chainer-raw-dc953-5bffc7f67f.18a8b41b069aee33 namespace: kserve-ci-e2e-test resourceVersion: "15233" uid: 3fe368cc-7599-4d9a-887d-3071a9edec16 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-dc953 namespace: kserve-ci-e2e-test resourceVersion: "15226" uid: abd98d0d-5d56-4447-acb5-0f2cdbfabe95 kind: Event lastTimestamp: "2026-04-22T14:28:01Z" message: Scaled up replica set model-chainer-raw-dc953-5bffc7f67f from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:28:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:28:01Z" name: model-chainer-raw-dc953.18a8b41b05353459 namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: 80b4bc25-8bdb-42e3-bffb-ff6b427e0ace reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:28:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-dc953 namespace: kserve-ci-e2e-test resourceVersion: "15245" uid: 69b0e178-75ca-4849-b73e-6553a1304954 kind: Event lastTimestamp: "2026-04-22T14:28:01Z" message: InferenceGraph [model-chainer-raw-dc953] is Ready metadata: creationTimestamp: "2026-04-22T14:28:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:28:01Z" name: model-chainer-raw-dc953.18a8b41b0c9675d1 namespace: kserve-ci-e2e-test resourceVersion: "15248" uid: 6f884cd2-6b94-4f35-b383-c49ad93eae4a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:29:41.504408Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-2ea51-76f765ddd7-ftp5b namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: 21106ddb-46e3-44d5-8124-34f5001722b5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-2ea51-76f765ddd7-ftp5b to ip-10-0-129-161.ec2.internal metadata: creationTimestamp: "2026-04-22T14:29:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:29:41Z" name: model-chainer-raw-hpa-2ea51-76f765ddd7-ftp5b.18a8b4326343e314 namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: e052a493-cb3d-44a3-ac0c-ff634154ecd4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56d6f9bdf8-db99f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:41Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-2ea51-76f765ddd7-ftp5b namespace: kserve-ci-e2e-test resourceVersion: "16206" uid: 21106ddb-46e3-44d5-8124-34f5001722b5 kind: Event lastTimestamp: "2026-04-22T14:29:41Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:29:41Z" name: model-chainer-raw-hpa-2ea51-76f765ddd7-ftp5b.18a8b4327bdc8592 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: 3c7aff18-2f6d-452e-afb8-78ce81a17890 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-2ea51} kind: Pod name: model-chainer-raw-hpa-2ea51-76f765ddd7-ftp5b namespace: kserve-ci-e2e-test resourceVersion: "16198" uid: 21106ddb-46e3-44d5-8124-34f5001722b5 kind: Event lastTimestamp: "2026-04-22T14:29:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:5ddf7b68668839b048ac1f65cf22f5a7b21a1ddc89705b9e0ad240d64f017b66" already present on machine metadata: creationTimestamp: "2026-04-22T14:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:29:41Z" name: model-chainer-raw-hpa-2ea51-76f765ddd7-ftp5b.18a8b4327d2a7c21 namespace: kserve-ci-e2e-test resourceVersion: "16219" uid: 9d0d5d0a-1043-459a-b18d-f6eae4a0af90 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-2ea51} kind: Pod name: model-chainer-raw-hpa-2ea51-76f765ddd7-ftp5b namespace: kserve-ci-e2e-test resourceVersion: "16198" uid: 21106ddb-46e3-44d5-8124-34f5001722b5 kind: Event lastTimestamp: "2026-04-22T14:29:41Z" message: 'Created container: model-chainer-raw-hpa-2ea51' metadata: creationTimestamp: "2026-04-22T14:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:29:41Z" name: model-chainer-raw-hpa-2ea51-76f765ddd7-ftp5b.18a8b4327fffffc3 namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: c328041b-66a0-43bf-95f3-b5618210bd91 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-2ea51} kind: Pod name: model-chainer-raw-hpa-2ea51-76f765ddd7-ftp5b namespace: kserve-ci-e2e-test resourceVersion: "16198" uid: 21106ddb-46e3-44d5-8124-34f5001722b5 kind: Event lastTimestamp: "2026-04-22T14:29:41Z" message: Started container model-chainer-raw-hpa-2ea51 metadata: creationTimestamp: "2026-04-22T14:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:29:41Z" name: model-chainer-raw-hpa-2ea51-76f765ddd7-ftp5b.18a8b432806eb25a namespace: kserve-ci-e2e-test resourceVersion: "16222" uid: a79c4992-062e-47ba-9299-dd4d0eb1ddf9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-2ea51} kind: Pod name: model-chainer-raw-hpa-2ea51-76f765ddd7-ftp5b namespace: kserve-ci-e2e-test resourceVersion: "16198" uid: 21106ddb-46e3-44d5-8124-34f5001722b5 kind: Event lastTimestamp: "2026-04-22T14:29:51Z" message: Stopping container model-chainer-raw-hpa-2ea51 metadata: creationTimestamp: "2026-04-22T14:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:29:51Z" name: model-chainer-raw-hpa-2ea51-76f765ddd7-ftp5b.18a8b434bb0fb913 namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: 8b781404-2220-4f89-af1f-40beb630bfb9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T14:29:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-2ea51} kind: Pod name: model-chainer-raw-hpa-2ea51-76f765ddd7-ftp5b namespace: kserve-ci-e2e-test resourceVersion: "16198" uid: 21106ddb-46e3-44d5-8124-34f5001722b5 kind: Event lastTimestamp: "2026-04-22T14:30:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T14:29:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:30:18Z" name: model-chainer-raw-hpa-2ea51-76f765ddd7-ftp5b.18a8b43516afb3de namespace: kserve-ci-e2e-test resourceVersion: "16663" uid: 2b6d75ca-fe86-4f64-b47d-4c1617801241 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-2ea51-76f765ddd7 namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: 057dd8a8-689b-470f-9a53-9f1840822c1e kind: Event lastTimestamp: "2026-04-22T14:29:41Z" message: 'Created pod: model-chainer-raw-hpa-2ea51-76f765ddd7-ftp5b' metadata: creationTimestamp: "2026-04-22T14:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:29:41Z" name: model-chainer-raw-hpa-2ea51-76f765ddd7.18a8b43262d4960d namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: a9e5ff68-13af-4242-882d-0808b94c09a1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-2ea51 namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: 59126e1c-1835-44ae-9449-991b93ff5bbe kind: Event lastTimestamp: "2026-04-22T14:29:41Z" message: Scaled up replica set model-chainer-raw-hpa-2ea51-76f765ddd7 from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:29:41Z" name: model-chainer-raw-hpa-2ea51.18a8b43262015e25 namespace: kserve-ci-e2e-test resourceVersion: "16193" uid: 0e717c46-aaa9-4fd9-bff3-79824514bdaf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:29:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-2ea51 namespace: kserve-ci-e2e-test resourceVersion: "16210" uid: 3bc2bbd0-bf63-4274-a667-79881cc1132a kind: Event lastTimestamp: "2026-04-22T14:29:41Z" message: InferenceGraph [model-chainer-raw-hpa-2ea51] is Ready metadata: creationTimestamp: "2026-04-22T14:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:29:41Z" name: model-chainer-raw-hpa-2ea51.18a8b4326760c08b namespace: kserve-ci-e2e-test resourceVersion: "16212" uid: c20d2831-76ec-458f-a5e7-4a02db99a6b5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:35:45.694340Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-77009-predictor-96b4df8df-bw2w4 namespace: kserve-ci-e2e-test resourceVersion: "19405" uid: 2be530f1-30b6-4733-8d04-01a05b638b74 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-77009-predictor-96b4df8df-bw2w4 to ip-10-0-129-161.ec2.internal metadata: creationTimestamp: "2026-04-22T14:35:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:35:45Z" name: raw-sklearn-77009-predictor-96b4df8df-bw2w4.18a8b4872ead3754 namespace: kserve-ci-e2e-test resourceVersion: "19411" uid: 3b8fb954-d35d-4967-bafa-e60b9e78f72d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56d6f9bdf8-db99f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:46Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-77009-predictor-96b4df8df-bw2w4 namespace: kserve-ci-e2e-test resourceVersion: "19419" uid: 2be530f1-30b6-4733-8d04-01a05b638b74 kind: Event lastTimestamp: "2026-04-22T14:35:46Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:35:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:35:46Z" name: raw-sklearn-77009-predictor-96b4df8df-bw2w4.18a8b48747317fe7 namespace: kserve-ci-e2e-test resourceVersion: "19426" uid: 36e4df42-2a34-4924-9380-d32b032ce872 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-77009-predictor-96b4df8df-bw2w4 namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: 2be530f1-30b6-4733-8d04-01a05b638b74 kind: Event lastTimestamp: "2026-04-22T14:35:46Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:498563386b50b61380307187059de966bc0a117e7161dd79d36731f0b2ebd43d" already present on machine metadata: creationTimestamp: "2026-04-22T14:35:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:46Z" name: raw-sklearn-77009-predictor-96b4df8df-bw2w4.18a8b487486572ac namespace: kserve-ci-e2e-test resourceVersion: "19428" uid: 3f79068f-3240-48ee-bd98-474af43cae8b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-77009-predictor-96b4df8df-bw2w4 namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: 2be530f1-30b6-4733-8d04-01a05b638b74 kind: Event lastTimestamp: "2026-04-22T14:35:46Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T14:35:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:46Z" name: raw-sklearn-77009-predictor-96b4df8df-bw2w4.18a8b4874b6ac53f namespace: kserve-ci-e2e-test resourceVersion: "19432" uid: 4e5fe4b4-7805-4c73-ad80-15267bcd1482 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-77009-predictor-96b4df8df-bw2w4 namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: 2be530f1-30b6-4733-8d04-01a05b638b74 kind: Event lastTimestamp: "2026-04-22T14:35:46Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T14:35:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:46Z" name: raw-sklearn-77009-predictor-96b4df8df-bw2w4.18a8b4874bdc3ae3 namespace: kserve-ci-e2e-test resourceVersion: "19433" uid: 227fc047-2c1d-4e0d-aec1-ba9e09b08d7d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-77009-predictor-96b4df8df-bw2w4 namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: 2be530f1-30b6-4733-8d04-01a05b638b74 kind: Event lastTimestamp: "2026-04-22T14:35:50Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:50Z" name: raw-sklearn-77009-predictor-96b4df8df-bw2w4.18a8b4883733957b namespace: kserve-ci-e2e-test resourceVersion: "19467" uid: bc67eef2-b557-413e-a46d-62d0a841f76f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-77009-predictor-96b4df8df-bw2w4 namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: 2be530f1-30b6-4733-8d04-01a05b638b74 kind: Event lastTimestamp: "2026-04-22T14:35:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:50Z" name: raw-sklearn-77009-predictor-96b4df8df-bw2w4.18a8b4883a29d62b namespace: kserve-ci-e2e-test resourceVersion: "19472" uid: 6f350467-8c90-495e-8cac-e6d86ce24be6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-77009-predictor-96b4df8df-bw2w4 namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: 2be530f1-30b6-4733-8d04-01a05b638b74 kind: Event lastTimestamp: "2026-04-22T14:35:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:35:50Z" name: raw-sklearn-77009-predictor-96b4df8df-bw2w4.18a8b4883a9d34da namespace: kserve-ci-e2e-test resourceVersion: "19473" uid: bdd903a3-7eb8-4188-9121-ee40a7a3b0d3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-22T14:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-77009-predictor-96b4df8df-bw2w4 namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: 2be530f1-30b6-4733-8d04-01a05b638b74 kind: Event lastTimestamp: "2026-04-22T14:36:52Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:36:52Z" name: raw-sklearn-77009-predictor-96b4df8df-bw2w4.18a8b488731834ce namespace: kserve-ci-e2e-test resourceVersion: "19909" uid: 8948f5d8-d04b-49ae-aba0-00771663b85c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-77009-predictor-96b4df8df-bw2w4 namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: 2be530f1-30b6-4733-8d04-01a05b638b74 kind: Event lastTimestamp: "2026-04-22T14:37:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:05Z" name: raw-sklearn-77009-predictor-96b4df8df-bw2w4.18a8b499d641172e namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: 7cd7c0f4-7192-435f-8deb-eb7dc19f694a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-77009-predictor-96b4df8df namespace: kserve-ci-e2e-test resourceVersion: "19396" uid: 3f5afc28-a976-425c-89d8-50a8f0f817ea kind: Event lastTimestamp: "2026-04-22T14:35:45Z" message: 'Created pod: raw-sklearn-77009-predictor-96b4df8df-bw2w4' metadata: creationTimestamp: "2026-04-22T14:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:35:45Z" name: raw-sklearn-77009-predictor-96b4df8df.18a8b4872e2b8dc8 namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: 2660c981-77b9-492e-ba7d-de88f104b699 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-77009-predictor namespace: kserve-ci-e2e-test resourceVersion: "19394" uid: 30e7aa48-9bce-4117-82d5-9a96eb2cdf8b kind: Event lastTimestamp: "2026-04-22T14:35:45Z" message: Scaled up replica set raw-sklearn-77009-predictor-96b4df8df from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:35:45Z" name: raw-sklearn-77009-predictor.18a8b4872bc2d420 namespace: kserve-ci-e2e-test resourceVersion: "19398" uid: ac4e5269-c63c-4780-bb26-aafbba9e83da reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-77009-predictor namespace: kserve-ci-e2e-test resourceVersion: "19395" uid: ea6f2c7c-7477-4985-b427-1455550d2add kind: Event lastTimestamp: "2026-04-22T14:36:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:36:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:00Z" name: raw-sklearn-77009-predictor.18a8b48aaa3ca3b8 namespace: kserve-ci-e2e-test resourceVersion: "19550" uid: 7505aedd-f632-40b8-9e44-f603da560db3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:36:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-77009-predictor namespace: kserve-ci-e2e-test resourceVersion: "19395" uid: ea6f2c7c-7477-4985-b427-1455550d2add kind: Event lastTimestamp: "2026-04-22T14:36:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:36:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:36:00Z" name: raw-sklearn-77009-predictor.18a8b48aaaa4d517 namespace: kserve-ci-e2e-test resourceVersion: "19552" uid: 80056b3c-a50f-482e-88ab-89f449539651 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T14:36:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-77009-predictor namespace: kserve-ci-e2e-test resourceVersion: "19551" uid: ea6f2c7c-7477-4985-b427-1455550d2add kind: Event lastTimestamp: "2026-04-22T14:37:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:36:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:37:00Z" name: raw-sklearn-77009-predictor.18a8b48e29b6a46a namespace: kserve-ci-e2e-test resourceVersion: "19968" uid: 65f53a3a-c964-47ba-a780-ef2a06cc266d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T14:36:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-77009-predictor namespace: kserve-ci-e2e-test resourceVersion: "19551" uid: ea6f2c7c-7477-4985-b427-1455550d2add kind: Event lastTimestamp: "2026-04-22T14:37:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:36:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:37:00Z" name: raw-sklearn-77009-predictor.18a8b48e2a1b6a8f namespace: kserve-ci-e2e-test resourceVersion: "19969" uid: ff3bbb53-4463-4ef8-807b-4978e7d0d351 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-22T14:35:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-77009 namespace: kserve-ci-e2e-test resourceVersion: "19384" uid: 96a60cb3-4a3c-404b-8af4-8aca4e0e3985 kind: Event lastTimestamp: "2026-04-22T14:37:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:37:02Z" name: raw-sklearn-77009.18a8b4872b9bb978 namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: 94e29451-9b17-4446-a111-aa193670813a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:35:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-77009 namespace: kserve-ci-e2e-test resourceVersion: "19384" uid: 96a60cb3-4a3c-404b-8af4-8aca4e0e3985 kind: Event lastTimestamp: "2026-04-22T14:35:45Z" message: 'Failed to update status for InferenceService "raw-sklearn-77009": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-77009": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:35:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:35:45Z" name: raw-sklearn-77009.18a8b4872bef83c0 namespace: kserve-ci-e2e-test resourceVersion: "19401" uid: 31c035df-8644-4cba-ba7d-fc471019fdd0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-77009 namespace: kserve-ci-e2e-test resourceVersion: "19987" uid: 96a60cb3-4a3c-404b-8af4-8aca4e0e3985 kind: Event lastTimestamp: "2026-04-22T14:37:02Z" message: InferenceService [raw-sklearn-77009] is Ready metadata: creationTimestamp: "2026-04-22T14:37:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:37:02Z" name: raw-sklearn-77009.18a8b498ff0559c7 namespace: kserve-ci-e2e-test resourceVersion: "19988" uid: 0668ebe2-aba7-4325-b00f-7d35eff4c18e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T14:37:05.933089Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2 namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: dba6cfee-31be-417c-89b2-ceb249d897d1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2 to ip-10-0-129-161.ec2.internal metadata: creationTimestamp: "2026-04-22T14:37:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T14:37:05Z" name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2.18a8b499dd475b5f namespace: kserve-ci-e2e-test resourceVersion: "20058" uid: d073b89b-a44c-4082-8a33-b20521e798f3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56d6f9bdf8-db99f source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:06Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2 namespace: kserve-ci-e2e-test resourceVersion: "20063" uid: dba6cfee-31be-417c-89b2-ceb249d897d1 kind: Event lastTimestamp: "2026-04-22T14:37:06Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T14:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T14:37:06Z" name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2.18a8b499f6462318 namespace: kserve-ci-e2e-test resourceVersion: "20069" uid: 364ebfc9-63e4-4c66-8bdf-64d38c88ec05 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2 namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: dba6cfee-31be-417c-89b2-ceb249d897d1 kind: Event lastTimestamp: "2026-04-22T14:37:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:498563386b50b61380307187059de966bc0a117e7161dd79d36731f0b2ebd43d" already present on machine metadata: creationTimestamp: "2026-04-22T14:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:06Z" name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2.18a8b499f76419ac namespace: kserve-ci-e2e-test resourceVersion: "20071" uid: 6c827007-4378-469a-8515-5ed301f7071c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2 namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: dba6cfee-31be-417c-89b2-ceb249d897d1 kind: Event lastTimestamp: "2026-04-22T14:37:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T14:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:06Z" name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2.18a8b499facb755a namespace: kserve-ci-e2e-test resourceVersion: "20073" uid: 1c500ca2-32b7-4750-95ea-5bb5ad6b1e73 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2 namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: dba6cfee-31be-417c-89b2-ceb249d897d1 kind: Event lastTimestamp: "2026-04-22T14:37:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T14:37:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:06Z" name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2.18a8b499fb3b5638 namespace: kserve-ci-e2e-test resourceVersion: "20074" uid: 3a915de4-0fd9-4415-b9d3-815212d11dd9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2 namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: dba6cfee-31be-417c-89b2-ceb249d897d1 kind: Event lastTimestamp: "2026-04-22T14:37:10Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1400" already present on machine metadata: creationTimestamp: "2026-04-22T14:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:10Z" name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2.18a8b49ae4ba08c7 namespace: kserve-ci-e2e-test resourceVersion: "20108" uid: 3eb2dd7c-bf90-4f73-a076-0a4e57e4d845 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2 namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: dba6cfee-31be-417c-89b2-ceb249d897d1 kind: Event lastTimestamp: "2026-04-22T14:37:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T14:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:10Z" name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2.18a8b49ae7d0c2c7 namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: 004efb82-01f2-41b7-8f3d-2d2ea3205c78 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2 namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: dba6cfee-31be-417c-89b2-ceb249d897d1 kind: Event lastTimestamp: "2026-04-22T14:37:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T14:37:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:37:10Z" name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2.18a8b49ae8426e5a namespace: kserve-ci-e2e-test resourceVersion: "20113" uid: 046b62d2-301f-4516-95c1-285f8b4ed849 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-22T14:37:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2 namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: dba6cfee-31be-417c-89b2-ceb249d897d1 kind: Event lastTimestamp: "2026-04-22T14:38:12Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T14:37:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:38:12Z" name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2.18a8b49b20aa43a0 namespace: kserve-ci-e2e-test resourceVersion: "20559" uid: 97fb5ade-af53-4fcc-ad6f-89cb13660be7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2 namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: dba6cfee-31be-417c-89b2-ceb249d897d1 kind: Event lastTimestamp: "2026-04-22T14:38:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T14:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T14:38:25Z" name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2.18a8b4ac80630fd4 namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: c05c6051-2cf0-4c18-a96c-9bd186edc2b5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-161.ec2.internal source: component: kubelet host: ip-10-0-129-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c namespace: kserve-ci-e2e-test resourceVersion: "20041" uid: 9c1c8cb7-9dc3-4190-aac2-b25002c36f51 kind: Event lastTimestamp: "2026-04-22T14:37:05Z" message: 'Created pod: raw-sklearn-runtime-9da2e-predictor-75cc5f46c-lg5g2' metadata: creationTimestamp: "2026-04-22T14:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:37:05Z" name: raw-sklearn-runtime-9da2e-predictor-75cc5f46c.18a8b499dcd06e58 namespace: kserve-ci-e2e-test resourceVersion: "20053" uid: d39240ce-81e7-4950-ae7d-3c6a9cc85af7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-9da2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: ee5353b8-d3fe-45c4-b4aa-e4a7ba1f1e49 kind: Event lastTimestamp: "2026-04-22T14:37:05Z" message: Scaled up replica set raw-sklearn-runtime-9da2e-predictor-75cc5f46c from 0 to 1 metadata: creationTimestamp: "2026-04-22T14:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:37:05Z" name: raw-sklearn-runtime-9da2e-predictor.18a8b499d8fd22bf namespace: kserve-ci-e2e-test resourceVersion: "20043" uid: 8faba993-0b34-44fb-a913-0b1ab5db1ba0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-9da2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 8a35ac50-96b7-4d36-848d-4507fa1210fe kind: Event lastTimestamp: "2026-04-22T14:37:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:37:20Z" name: raw-sklearn-runtime-9da2e-predictor.18a8b49d58170b0c namespace: kserve-ci-e2e-test resourceVersion: "20192" uid: 217e4811-e5f2-495c-b883-81b11de3def3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-9da2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 8a35ac50-96b7-4d36-848d-4507fa1210fe kind: Event lastTimestamp: "2026-04-22T14:37:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T14:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:37:20Z" name: raw-sklearn-runtime-9da2e-predictor.18a8b49d587a65e0 namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: 5a63f43a-82a3-4222-816e-74589474bdfa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T14:37:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-9da2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20193" uid: 8a35ac50-96b7-4d36-848d-4507fa1210fe kind: Event lastTimestamp: "2026-04-22T14:38:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:38:20Z" name: raw-sklearn-runtime-9da2e-predictor.18a8b4a0d72e717f namespace: kserve-ci-e2e-test resourceVersion: "20621" uid: ef344165-9fea-4db3-87cc-6d0aaca48be4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T14:37:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-9da2e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20193" uid: 8a35ac50-96b7-4d36-848d-4507fa1210fe kind: Event lastTimestamp: "2026-04-22T14:38:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T14:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T14:38:20Z" name: raw-sklearn-runtime-9da2e-predictor.18a8b4a0d795e63c namespace: kserve-ci-e2e-test resourceVersion: "20622" uid: a3b9fc60-4a76-4b8b-ad0d-222b430fbf80 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-22T14:37:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-9da2e namespace: kserve-ci-e2e-test resourceVersion: "20015" uid: 5e1c8791-79e1-4b32-901e-4200bceadecb kind: Event lastTimestamp: "2026-04-22T14:38:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T14:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:38:22Z" name: raw-sklearn-runtime-9da2e.18a8b499d914d350 namespace: kserve-ci-e2e-test resourceVersion: "20639" uid: 264a3c05-bd34-41e1-971c-7c8333c8f819 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:37:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-9da2e namespace: kserve-ci-e2e-test resourceVersion: "20015" uid: 5e1c8791-79e1-4b32-901e-4200bceadecb kind: Event lastTimestamp: "2026-04-22T14:37:05Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-9da2e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-9da2e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T14:37:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:37:05Z" name: raw-sklearn-runtime-9da2e.18a8b499da161a62 namespace: kserve-ci-e2e-test resourceVersion: "20047" uid: 5d68ac10-98c1-4dfe-b48b-45929032d52a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T14:38:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-9da2e namespace: kserve-ci-e2e-test resourceVersion: "20637" uid: 5e1c8791-79e1-4b32-901e-4200bceadecb kind: Event lastTimestamp: "2026-04-22T14:38:22Z" message: InferenceService [raw-sklearn-runtime-9da2e] is Ready metadata: creationTimestamp: "2026-04-22T14:38:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T14:38:22Z" name: raw-sklearn-runtime-9da2e.18a8b4abad0d1e1f namespace: kserve-ci-e2e-test resourceVersion: "20638" uid: 0f124931-7596-4c0b-95ea-19edafe787a9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "20868"