--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:25:47.299504Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-bf4c24-predictor-7d468db94-rhw7x namespace: kserve-ci-e2e-test resourceVersion: "19123" uid: be8bc7bd-935c-4317-9d0c-acdbc9d2f5de kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-bf4c24-predictor-7d468db94-rhw7x to ip-10-0-138-52.ec2.internal metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:25:47Z" name: isvc-init-fail-bf4c24-predictor-7d468db94-rhw7x.18a96183f1c9b15f namespace: kserve-ci-e2e-test resourceVersion: "19127" uid: e43ced6b-ce44-4ad9-909e-e6ccb7ce69c0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df6b44bbb-hm2bm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-bf4c24-predictor-7d468db94-rhw7x namespace: kserve-ci-e2e-test resourceVersion: "19132" uid: be8bc7bd-935c-4317-9d0c-acdbc9d2f5de kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: Add eth0 [10.132.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:25:47Z" name: isvc-init-fail-bf4c24-predictor-7d468db94-rhw7x.18a961840ba92f53 namespace: kserve-ci-e2e-test resourceVersion: "19140" uid: 4635a9cb-4344-4304-b84a-f455c7747707 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-bf4c24-predictor-7d468db94-rhw7x namespace: kserve-ci-e2e-test resourceVersion: "19126" uid: be8bc7bd-935c-4317-9d0c-acdbc9d2f5de kind: Event lastTimestamp: "2026-04-24T19:25:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0183e14c0f4ec83676f700a2c092ab4e3599fa115c6cab4546b23e5207166882" already present on machine metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:52Z" name: isvc-init-fail-bf4c24-predictor-7d468db94-rhw7x.18a961840d1a9067 namespace: kserve-ci-e2e-test resourceVersion: "19185" uid: 3a5a609b-6133-4327-b9f4-63861528fb0e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-bf4c24-predictor-7d468db94-rhw7x namespace: kserve-ci-e2e-test resourceVersion: "19126" uid: be8bc7bd-935c-4317-9d0c-acdbc9d2f5de kind: Event lastTimestamp: "2026-04-24T19:25:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:52Z" name: isvc-init-fail-bf4c24-predictor-7d468db94-rhw7x.18a961841133cb21 namespace: kserve-ci-e2e-test resourceVersion: "19191" uid: 22a8c689-20cd-4773-b2a5-8781cf1c866e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-bf4c24-predictor-7d468db94-rhw7x namespace: kserve-ci-e2e-test resourceVersion: "19126" uid: be8bc7bd-935c-4317-9d0c-acdbc9d2f5de kind: Event lastTimestamp: "2026-04-24T19:25:52Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:52Z" name: isvc-init-fail-bf4c24-predictor-7d468db94-rhw7x.18a9618411ade578 namespace: kserve-ci-e2e-test resourceVersion: "19193" uid: 8a76b92b-a650-4b37-8f23-23b8897fea44 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-bf4c24-predictor-7d468db94-rhw7x namespace: kserve-ci-e2e-test resourceVersion: "19126" uid: be8bc7bd-935c-4317-9d0c-acdbc9d2f5de kind: Event lastTimestamp: "2026-04-24T19:25:53Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-04-24T19:25:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:53Z" name: isvc-init-fail-bf4c24-predictor-7d468db94-rhw7x.18a96185462d72f9 namespace: kserve-ci-e2e-test resourceVersion: "19260" uid: bdaab99e-17ce-4e97-b0f8-cfd7cca92b6b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-bf4c24-predictor-7d468db94 namespace: kserve-ci-e2e-test resourceVersion: "19111" uid: e6737066-5cee-4fd5-ba11-2609355ddd1a kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: 'Created pod: isvc-init-fail-bf4c24-predictor-7d468db94-rhw7x' metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:47Z" name: isvc-init-fail-bf4c24-predictor-7d468db94.18a96183f12872aa namespace: kserve-ci-e2e-test resourceVersion: "19124" uid: f34b8d0c-6f01-4c7a-ba37-5430026702e6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-bf4c24-predictor namespace: kserve-ci-e2e-test resourceVersion: "19107" uid: 3c989ab8-7a98-45ce-b9ff-0cfe9e4125a1 kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: Scaled up replica set isvc-init-fail-bf4c24-predictor-7d468db94 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:47Z" name: isvc-init-fail-bf4c24-predictor.18a96183ed12ccd6 namespace: kserve-ci-e2e-test resourceVersion: "19115" uid: e076befa-963c-4661-8e0b-03430ad819b0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-bf4c24 namespace: kserve-ci-e2e-test resourceVersion: "19084" uid: 31054841-43b9-4b32-82c3-c44c035d5e7c kind: Event lastTimestamp: "2026-04-24T19:25:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:25:52Z" name: isvc-init-fail-bf4c24.18a96183ecdca265 namespace: kserve-ci-e2e-test resourceVersion: "19192" uid: bef6cf1e-e332-4bdc-8732-a1b924f1addb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-bf4c24 namespace: kserve-ci-e2e-test resourceVersion: "19084" uid: 31054841-43b9-4b32-82c3-c44c035d5e7c kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: 'Failed to update status for InferenceService "isvc-init-fail-bf4c24": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-bf4c24": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:25:47Z" name: isvc-init-fail-bf4c24.18a96183edb3cd9e namespace: kserve-ci-e2e-test resourceVersion: "19117" uid: b1ca30d9-5151-4137-b007-1c99a90b3cf6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-bf4c24 namespace: kserve-ci-e2e-test resourceVersion: "19084" uid: 31054841-43b9-4b32-82c3-c44c035d5e7c kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-bf4c24": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:25:47Z" name: isvc-init-fail-bf4c24.18a96183edb438cc namespace: kserve-ci-e2e-test resourceVersion: "19119" uid: f6748dd0-2382-4d6e-b766-d645deb3e007 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:20:15.380297Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 namespace: kserve-ci-e2e-test resourceVersion: "16361" uid: 79dffc7a-0681-4abb-ba23-01470dfb0cec kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 to ip-10-0-138-52.ec2.internal metadata: creationTimestamp: "2026-04-24T19:20:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:20:15Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4.18a96136a9dc8ea3 namespace: kserve-ci-e2e-test resourceVersion: "16366" uid: 27a6b926-3acc-4b34-ba08-d86b79953b09 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df6b44bbb-hm2bm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:20:15Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 namespace: kserve-ci-e2e-test resourceVersion: "16370" uid: 79dffc7a-0681-4abb-ba23-01470dfb0cec kind: Event lastTimestamp: "2026-04-24T19:20:15Z" message: Add eth0 [10.132.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:20:15Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4.18a96136c3a37362 namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: bb2a6f09-1d88-4e27-89e9-285bedbb816e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:20:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 79dffc7a-0681-4abb-ba23-01470dfb0cec kind: Event lastTimestamp: "2026-04-24T19:20:15Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0183e14c0f4ec83676f700a2c092ab4e3599fa115c6cab4546b23e5207166882" already present on machine metadata: creationTimestamp: "2026-04-24T19:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:20:15Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4.18a96136c538787c namespace: kserve-ci-e2e-test resourceVersion: "16382" uid: 55109098-7619-447a-9e1f-5a750b08f553 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:20:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 79dffc7a-0681-4abb-ba23-01470dfb0cec kind: Event lastTimestamp: "2026-04-24T19:20:15Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T19:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:20:15Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4.18a96136c8f4a7db namespace: kserve-ci-e2e-test resourceVersion: "16383" uid: b7145de7-d2e8-4390-a0d7-31ea88457cd6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:20:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 79dffc7a-0681-4abb-ba23-01470dfb0cec kind: Event lastTimestamp: "2026-04-24T19:20:15Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T19:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:20:15Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4.18a96136c96803fc namespace: kserve-ci-e2e-test resourceVersion: "16384" uid: ff36c379-a147-4070-ad48-09d84edb5064 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:20:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 79dffc7a-0681-4abb-ba23-01470dfb0cec kind: Event lastTimestamp: "2026-04-24T19:20:19Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:20:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:20:19Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4.18a96137ae0e58e9 namespace: kserve-ci-e2e-test resourceVersion: "16419" uid: ffe80171-a6e6-4cbc-92d7-b43d558f3677 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:20:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 79dffc7a-0681-4abb-ba23-01470dfb0cec kind: Event lastTimestamp: "2026-04-24T19:20:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:20:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:20:19Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4.18a96137b2b9471b namespace: kserve-ci-e2e-test resourceVersion: "16424" uid: 701e6f13-674b-4909-a8a4-24f29aac4ffb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:20:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 79dffc7a-0681-4abb-ba23-01470dfb0cec kind: Event lastTimestamp: "2026-04-24T19:20:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:20:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:20:19Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4.18a96137b347f9e4 namespace: kserve-ci-e2e-test resourceVersion: "16425" uid: a6b9f74d-e2a1-4648-96d0-712e94cd1246 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:20:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 79dffc7a-0681-4abb-ba23-01470dfb0cec kind: Event lastTimestamp: "2026-04-24T19:20:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:20:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:20:19Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4.18a96137b3580028 namespace: kserve-ci-e2e-test resourceVersion: "16426" uid: 938a1a6d-ee38-49b5-a9b8-8f7ce3019fb4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:20:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 79dffc7a-0681-4abb-ba23-01470dfb0cec kind: Event lastTimestamp: "2026-04-24T19:20:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:20:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:20:19Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4.18a96137b70ad372 namespace: kserve-ci-e2e-test resourceVersion: "16427" uid: 7015db18-73e7-4063-afae-c563bfca03f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:20:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 79dffc7a-0681-4abb-ba23-01470dfb0cec kind: Event lastTimestamp: "2026-04-24T19:20:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:20:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:20:19Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4.18a96137b786d678 namespace: kserve-ci-e2e-test resourceVersion: "16428" uid: bc6beef6-77d3-4d6b-ac4b-b0f78e8d8095 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:20:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 79dffc7a-0681-4abb-ba23-01470dfb0cec kind: Event lastTimestamp: "2026-04-24T19:20:19Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:d8f23d76f2d9886b9b5bc998ebdde5dc6c174340fd697aa08b5f7fbfe1b78fbb" already present on machine metadata: creationTimestamp: "2026-04-24T19:20:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:20:19Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4.18a96137b7936ebb namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: a8efcb9a-41d5-43f4-9bcf-c2323e1d1adf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:20:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 79dffc7a-0681-4abb-ba23-01470dfb0cec kind: Event lastTimestamp: "2026-04-24T19:20:19Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-24T19:20:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:20:19Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4.18a96137bb19fbcb namespace: kserve-ci-e2e-test resourceVersion: "16430" uid: 96632fd3-dd37-4d75-83b0-0ac4f4d9f063 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:20:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 79dffc7a-0681-4abb-ba23-01470dfb0cec kind: Event lastTimestamp: "2026-04-24T19:20:19Z" message: Started container agent metadata: creationTimestamp: "2026-04-24T19:20:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:20:19Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4.18a96137bba58bf0 namespace: kserve-ci-e2e-test resourceVersion: "16431" uid: 85385ec5-1af9-42bb-9a79-1eb3c707c6dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-24T19:20:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 79dffc7a-0681-4abb-ba23-01470dfb0cec kind: Event lastTimestamp: "2026-04-24T19:21:57Z" message: 'Readiness probe failed: dial tcp 10.132.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:21:57Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4.18a96137ea14b5ca namespace: kserve-ci-e2e-test resourceVersion: "17264" uid: b734400a-0fa1-4894-b6db-8fdebf45ecb9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T19:20:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 79dffc7a-0681-4abb-ba23-01470dfb0cec kind: Event lastTimestamp: "2026-04-24T19:21:57Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T19:20:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:21:57Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4.18a9613825da8bcc namespace: kserve-ci-e2e-test resourceVersion: "17265" uid: 4cc86b96-b770-479c-9ad1-f06f278e9a8e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:21:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 79dffc7a-0681-4abb-ba23-01470dfb0cec kind: Event lastTimestamp: "2026-04-24T19:21:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:21:40Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4.18a9614a80e0335f namespace: kserve-ci-e2e-test resourceVersion: "17068" uid: 8042eb7f-ceba-4f20-9f41-6cf8f47c10d5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:21:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 79dffc7a-0681-4abb-ba23-01470dfb0cec kind: Event lastTimestamp: "2026-04-24T19:21:40Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-24T19:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:21:40Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4.18a9614a80e06f22 namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: 39ef2689-991e-4f6f-be2c-cbfc1d0b36e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:21:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 79dffc7a-0681-4abb-ba23-01470dfb0cec kind: Event lastTimestamp: "2026-04-24T19:21:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:21:40Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4.18a9614a80e115a5 namespace: kserve-ci-e2e-test resourceVersion: "17070" uid: eef1188f-c0e1-408e-932f-3fdf73ce4852 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T19:21:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4 namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: 79dffc7a-0681-4abb-ba23-01470dfb0cec kind: Event lastTimestamp: "2026-04-24T19:21:57Z" message: 'Readiness probe failed: Get "https://10.132.0.45:8643/healthz": dial tcp 10.132.0.45:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:21:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:21:57Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4.18a9614b0206ab53 namespace: kserve-ci-e2e-test resourceVersion: "17263" uid: ec1f3bb9-0e17-4631-b61d-96956bd214cf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:20:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-2ea81-predictor-664ffb7f86 namespace: kserve-ci-e2e-test resourceVersion: "16351" uid: 3f0508f5-5bf2-452c-a874-02bf77fa363e kind: Event lastTimestamp: "2026-04-24T19:20:15Z" message: 'Created pod: isvc-logger-raw-2ea81-predictor-664ffb7f86-k5mr4' metadata: creationTimestamp: "2026-04-24T19:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:20:15Z" name: isvc-logger-raw-2ea81-predictor-664ffb7f86.18a96136a9446641 namespace: kserve-ci-e2e-test resourceVersion: "16363" uid: e5406ea4-8327-4f1e-8039-5c57c1f0e796 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:20:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-2ea81-predictor namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: 1ba58c22-509a-4f10-b2ca-51d2dbd190b5 kind: Event lastTimestamp: "2026-04-24T19:20:15Z" message: Scaled up replica set isvc-logger-raw-2ea81-predictor-664ffb7f86 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:20:15Z" name: isvc-logger-raw-2ea81-predictor.18a96136a63c976c namespace: kserve-ci-e2e-test resourceVersion: "16353" uid: 8178e30d-4c36-4b22-a716-bc8728f3645d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:20:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-2ea81-predictor namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: a4a326e1-bbb8-4f81-8347-bd758720fde7 kind: Event lastTimestamp: "2026-04-24T19:20:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:20:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:20:45Z" name: isvc-logger-raw-2ea81-predictor.18a9613a24e8f705 namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 5d904b66-640d-4f7d-a03d-8f98077f8957 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:20:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-2ea81-predictor namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: a4a326e1-bbb8-4f81-8347-bd758720fde7 kind: Event lastTimestamp: "2026-04-24T19:20:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:20:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:20:45Z" name: isvc-logger-raw-2ea81-predictor.18a9613a2571f1d1 namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: b0c91b82-3638-4bc9-9cfd-0600a4501516 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:21:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-2ea81-predictor namespace: kserve-ci-e2e-test resourceVersion: "16525" uid: a4a326e1-bbb8-4f81-8347-bd758720fde7 kind: Event lastTimestamp: "2026-04-24T19:21:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:21:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:21:30Z" name: isvc-logger-raw-2ea81-predictor.18a9614122efa655 namespace: kserve-ci-e2e-test resourceVersion: "16958" uid: bb0565d9-776a-417a-b1e7-d7fa7d671da4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:21:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-2ea81-predictor namespace: kserve-ci-e2e-test resourceVersion: "16525" uid: a4a326e1-bbb8-4f81-8347-bd758720fde7 kind: Event lastTimestamp: "2026-04-24T19:21:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:21:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:21:30Z" name: isvc-logger-raw-2ea81-predictor.18a96141236638a4 namespace: kserve-ci-e2e-test resourceVersion: "16959" uid: 464d295f-676d-4fb8-9903-b701f2a345ba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T19:20:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-2ea81 namespace: kserve-ci-e2e-test resourceVersion: "16338" uid: 25f77059-cf4c-4652-ae37-398de0ee455f kind: Event lastTimestamp: "2026-04-24T19:21:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:21:27Z" name: isvc-logger-raw-2ea81.18a96136a63b1855 namespace: kserve-ci-e2e-test resourceVersion: "16942" uid: 903b2681-3ada-4703-a5e3-a259d4ba0c94 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:20:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-2ea81 namespace: kserve-ci-e2e-test resourceVersion: "16338" uid: 25f77059-cf4c-4652-ae37-398de0ee455f kind: Event lastTimestamp: "2026-04-24T19:20:15Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-2ea81": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-2ea81": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:20:15Z" name: isvc-logger-raw-2ea81.18a96136a6c6057a namespace: kserve-ci-e2e-test resourceVersion: "16355" uid: d03c91d3-89e8-44b3-9835-0469f154288e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:21:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-2ea81 namespace: kserve-ci-e2e-test resourceVersion: "16940" uid: 25f77059-cf4c-4652-ae37-398de0ee455f kind: Event lastTimestamp: "2026-04-24T19:21:27Z" message: InferenceService [isvc-logger-raw-2ea81] is Ready metadata: creationTimestamp: "2026-04-24T19:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:21:27Z" name: isvc-logger-raw-2ea81.18a9614789e9b874 namespace: kserve-ci-e2e-test resourceVersion: "16941" uid: 738b963d-2433-44be-9217-a482febf32cf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:24:10.889622Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b namespace: kserve-ci-e2e-test resourceVersion: "18252" uid: a60bdf07-5eab-4ed3-9e8a-3f0f42af0a86 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b to ip-10-0-138-52.ec2.internal metadata: creationTimestamp: "2026-04-24T19:24:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:24:10Z" name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b.18a9616d7f4fa5e7 namespace: kserve-ci-e2e-test resourceVersion: "18257" uid: 67de726a-c2c3-4413-b5ae-488f144f8da1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df6b44bbb-hm2bm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a60bdf07-5eab-4ed3-9e8a-3f0f42af0a86 kind: Event lastTimestamp: "2026-04-24T19:24:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-primary-b4ed0b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T19:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:11Z" name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b.18a9616d8b6d5feb namespace: kserve-ci-e2e-test resourceVersion: "18266" uid: abbb277b-f87a-4d14-a432-07519cff7edc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b namespace: kserve-ci-e2e-test resourceVersion: "18264" uid: a60bdf07-5eab-4ed3-9e8a-3f0f42af0a86 kind: Event lastTimestamp: "2026-04-24T19:24:11Z" message: Add eth0 [10.132.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:24:11Z" name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b.18a9616dbc949629 namespace: kserve-ci-e2e-test resourceVersion: "18276" uid: 54f0bbd5-bf98-4aa3-a0b4-55f8e4e04a93 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a60bdf07-5eab-4ed3-9e8a-3f0f42af0a86 kind: Event lastTimestamp: "2026-04-24T19:24:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0183e14c0f4ec83676f700a2c092ab4e3599fa115c6cab4546b23e5207166882" already present on machine metadata: creationTimestamp: "2026-04-24T19:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:11Z" name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b.18a9616dbe18c666 namespace: kserve-ci-e2e-test resourceVersion: "18279" uid: 723add68-817f-48f2-a85e-f70284964e0c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a60bdf07-5eab-4ed3-9e8a-3f0f42af0a86 kind: Event lastTimestamp: "2026-04-24T19:24:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T19:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:11Z" name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b.18a9616dc1569965 namespace: kserve-ci-e2e-test resourceVersion: "18280" uid: eae5d28a-e4d5-4791-ab42-3def6039e223 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a60bdf07-5eab-4ed3-9e8a-3f0f42af0a86 kind: Event lastTimestamp: "2026-04-24T19:24:12Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T19:24:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:12Z" name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b.18a9616dc1ce5ff4 namespace: kserve-ci-e2e-test resourceVersion: "18281" uid: f6c8129c-9872-41ce-95d4-e560de252b6d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a60bdf07-5eab-4ed3-9e8a-3f0f42af0a86 kind: Event lastTimestamp: "2026-04-24T19:24:16Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:16Z" name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b.18a9616ed64be3ab namespace: kserve-ci-e2e-test resourceVersion: "18324" uid: 5137f105-87a3-4ab5-b1b2-d723af361ab4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a60bdf07-5eab-4ed3-9e8a-3f0f42af0a86 kind: Event lastTimestamp: "2026-04-24T19:24:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:16Z" name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b.18a9616eda488799 namespace: kserve-ci-e2e-test resourceVersion: "18329" uid: 98f604ee-caff-44b3-b718-6fc01c8911c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a60bdf07-5eab-4ed3-9e8a-3f0f42af0a86 kind: Event lastTimestamp: "2026-04-24T19:24:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:16Z" name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b.18a9616edac3055f namespace: kserve-ci-e2e-test resourceVersion: "18330" uid: b018c127-4e87-460e-a5a3-4d2fad66b3e3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a60bdf07-5eab-4ed3-9e8a-3f0f42af0a86 kind: Event lastTimestamp: "2026-04-24T19:24:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:16Z" name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b.18a9616edad0ee06 namespace: kserve-ci-e2e-test resourceVersion: "18332" uid: 7bf5673a-3383-4142-bdfd-bcefdbc90184 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a60bdf07-5eab-4ed3-9e8a-3f0f42af0a86 kind: Event lastTimestamp: "2026-04-24T19:24:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:16Z" name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b.18a9616edeaccff1 namespace: kserve-ci-e2e-test resourceVersion: "18333" uid: 73178401-9b88-4ee8-9c90-6912d8467c74 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a60bdf07-5eab-4ed3-9e8a-3f0f42af0a86 kind: Event lastTimestamp: "2026-04-24T19:24:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:16Z" name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b.18a9616edf5635bd namespace: kserve-ci-e2e-test resourceVersion: "18334" uid: eabddb02-ba68-4400-8594-8df924a9e179 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T19:24:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a60bdf07-5eab-4ed3-9e8a-3f0f42af0a86 kind: Event lastTimestamp: "2026-04-24T19:25:14Z" message: 'Readiness probe failed: dial tcp 10.132.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:24:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:14Z" name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b.18a9616f4e04541c namespace: kserve-ci-e2e-test resourceVersion: "18754" uid: c3c29bdb-448f-4147-8b5e-6f415fd871c5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a60bdf07-5eab-4ed3-9e8a-3f0f42af0a86 kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:47Z" name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b.18a96183ecff519c namespace: kserve-ci-e2e-test resourceVersion: "19114" uid: 670e74f6-09e2-4047-bb5d-12aabee9529b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a60bdf07-5eab-4ed3-9e8a-3f0f42af0a86 kind: Event lastTimestamp: "2026-04-24T19:25:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:25:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:47Z" name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b.18a96183ecffefaa namespace: kserve-ci-e2e-test resourceVersion: "19118" uid: e6dc2ea4-76e6-47de-b491-d9f8ad86996a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b namespace: kserve-ci-e2e-test resourceVersion: "18254" uid: a60bdf07-5eab-4ed3-9e8a-3f0f42af0a86 kind: Event lastTimestamp: "2026-04-24T19:25:49Z" message: 'Readiness probe failed: Get "https://10.132.0.47:8643/healthz": dial tcp 10.132.0.47:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:25:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:49Z" name: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b.18a961847e34c4c9 namespace: kserve-ci-e2e-test resourceVersion: "19168" uid: 198c4ff7-b3ea-4966-919b-c10e8f8fa468 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-b4ed0b-predictor-5fdfb76759 namespace: kserve-ci-e2e-test resourceVersion: "18241" uid: 27fd27ea-78e4-41c0-99e3-654b1225b271 kind: Event lastTimestamp: "2026-04-24T19:24:10Z" message: 'Created pod: isvc-primary-b4ed0b-predictor-5fdfb76759-zh94b' metadata: creationTimestamp: "2026-04-24T19:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:24:10Z" name: isvc-primary-b4ed0b-predictor-5fdfb76759.18a9616d7ee2ed53 namespace: kserve-ci-e2e-test resourceVersion: "18253" uid: b8fa1e78-3792-48b4-b078-e175fe010eed reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-b4ed0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18238" uid: 71090374-dbef-48b5-b2d2-c8567f7a9ff6 kind: Event lastTimestamp: "2026-04-24T19:24:10Z" message: Scaled up replica set isvc-primary-b4ed0b-predictor-5fdfb76759 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:24:10Z" name: isvc-primary-b4ed0b-predictor.18a9616d7aca9650 namespace: kserve-ci-e2e-test resourceVersion: "18243" uid: 4895359d-5a03-42cc-bc29-d3a99fa98f1c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:24:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-b4ed0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18240" uid: e492b0c3-2266-43a6-b467-2d248eec3430 kind: Event lastTimestamp: "2026-04-24T19:24:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:24:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:24:55Z" name: isvc-primary-b4ed0b-predictor.18a96170f9dcb5a1 namespace: kserve-ci-e2e-test resourceVersion: "18622" uid: 773fb63e-f958-4520-842c-a9ef119dbdd4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:24:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-b4ed0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18240" uid: e492b0c3-2266-43a6-b467-2d248eec3430 kind: Event lastTimestamp: "2026-04-24T19:24:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:24:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:24:55Z" name: isvc-primary-b4ed0b-predictor.18a96170fa44f6e9 namespace: kserve-ci-e2e-test resourceVersion: "18623" uid: 193cda69-f56a-4cf1-948c-84a2e8662bdf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:25:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-b4ed0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18411" uid: e492b0c3-2266-43a6-b467-2d248eec3430 kind: Event lastTimestamp: "2026-04-24T19:25:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:40Z" name: isvc-primary-b4ed0b-predictor.18a9617b7777bae2 namespace: kserve-ci-e2e-test resourceVersion: "19007" uid: 2c965d4b-6a69-4d91-a9ee-4d0bf6666182 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:25:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-b4ed0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18411" uid: e492b0c3-2266-43a6-b467-2d248eec3430 kind: Event lastTimestamp: "2026-04-24T19:25:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:25:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:40Z" name: isvc-primary-b4ed0b-predictor.18a9617b77e3aabb namespace: kserve-ci-e2e-test resourceVersion: "19008" uid: 7ec699bd-69c2-4eb5-aff2-3d16fd58d9db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-24T19:24:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-b4ed0b namespace: kserve-ci-e2e-test resourceVersion: "18213" uid: 49672e11-fcef-4df0-a57a-bed1fd78d31e kind: Event lastTimestamp: "2026-04-24T19:25:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:25:24Z" name: isvc-primary-b4ed0b.18a9616d7aa26173 namespace: kserve-ci-e2e-test resourceVersion: "18831" uid: 3d6ae298-88d0-457c-bd59-3d4391e68d4d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:24:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-b4ed0b namespace: kserve-ci-e2e-test resourceVersion: "18213" uid: 49672e11-fcef-4df0-a57a-bed1fd78d31e kind: Event lastTimestamp: "2026-04-24T19:24:10Z" message: 'Failed to update status for InferenceService "isvc-primary-b4ed0b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-b4ed0b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:24:10Z" name: isvc-primary-b4ed0b.18a9616d7af6176c namespace: kserve-ci-e2e-test resourceVersion: "18258" uid: fc6d3e7e-85ea-4c31-a9fb-da811b00ead2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:24:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-b4ed0b namespace: kserve-ci-e2e-test resourceVersion: "18213" uid: 49672e11-fcef-4df0-a57a-bed1fd78d31e kind: Event lastTimestamp: "2026-04-24T19:24:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-b4ed0b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:24:10Z" name: isvc-primary-b4ed0b.18a9616d7af66404 namespace: kserve-ci-e2e-test resourceVersion: "18261" uid: 0d17ca47-e0eb-42fa-bcf7-4189e70e3bf9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-b4ed0b namespace: kserve-ci-e2e-test resourceVersion: "18829" uid: 49672e11-fcef-4df0-a57a-bed1fd78d31e kind: Event lastTimestamp: "2026-04-24T19:25:24Z" message: InferenceService [isvc-primary-b4ed0b] is Ready metadata: creationTimestamp: "2026-04-24T19:25:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:25:24Z" name: isvc-primary-b4ed0b.18a9617eb05aaee8 namespace: kserve-ci-e2e-test resourceVersion: "18830" uid: 353789a0-d089-4ac5-9dce-509edbd53f7a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:14:39.673931Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13280" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w to ip-10-0-138-52.ec2.internal metadata: creationTimestamp: "2026-04-24T19:14:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:14:39Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a960e8803448b5 namespace: kserve-ci-e2e-test resourceVersion: "13285" uid: 82f52f85-1fc9-4811-8f43-2cd41fb0a875 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df6b44bbb-hm2bm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13290" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:14:40Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:14:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:14:40Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a960e89a499d42 namespace: kserve-ci-e2e-test resourceVersion: "13298" uid: 247a1ef9-ee7f-4439-92af-d1f61753b551 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:40Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:14:40Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0183e14c0f4ec83676f700a2c092ab4e3599fa115c6cab4546b23e5207166882" metadata: creationTimestamp: "2026-04-24T19:14:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:40Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a960e89bf221f4 namespace: kserve-ci-e2e-test resourceVersion: "13300" uid: 7df0113e-ef63-4421-9d32-ef16bd329fdf reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:14:43Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0183e14c0f4ec83676f700a2c092ab4e3599fa115c6cab4546b23e5207166882" in 3.439s (3.439s including waiting). Image size: 301492123 bytes.' metadata: creationTimestamp: "2026-04-24T19:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:43Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a960e968f5bddd namespace: kserve-ci-e2e-test resourceVersion: "13327" uid: 042e8319-acec-4113-9acc-a5ad88c19f3b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:14:43Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T19:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:43Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a960e96ceb8b44 namespace: kserve-ci-e2e-test resourceVersion: "13328" uid: 6d3bb3a3-3f61-40c5-8bbc-0314e5de5d4f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:14:43Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T19:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:43Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a960e96d60cb4c namespace: kserve-ci-e2e-test resourceVersion: "13329" uid: 5d802d75-bec0-4a41-b901-1b190766882f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:14:47Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1445" metadata: creationTimestamp: "2026-04-24T19:14:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:14:47Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a960ea4deb8b3e namespace: kserve-ci-e2e-test resourceVersion: "13362" uid: 3fd83e31-c290-4347-bbb5-e5a338fe315a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:15:00Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1445" in 13.525s (13.525s including waiting). Image size: 1560924583 bytes.' metadata: creationTimestamp: "2026-04-24T19:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:00Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a960ed741da085 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: 09ea65ca-b494-42e2-8bb1-1c44a6579800 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:15:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:01Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a960ed77813fc8 namespace: kserve-ci-e2e-test resourceVersion: "13465" uid: ad337071-7543-44a2-8272-40aeff58c281 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:15:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:01Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a960ed77fa74f1 namespace: kserve-ci-e2e-test resourceVersion: "13466" uid: 5af35d09-888a-4664-8e45-9aaf855781e5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:15:01Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T19:15:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:01Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a960ed7807d436 namespace: kserve-ci-e2e-test resourceVersion: "13467" uid: fce3eae9-4457-4145-b4c2-9d4e20946bcd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:15:03Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.138s (2.138s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T19:15:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:03Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a960edf785f0d7 namespace: kserve-ci-e2e-test resourceVersion: "13482" uid: a78118c8-32bb-4252-b70d-0781d3fdd132 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:15:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:15:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:03Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a960edfc2e82f9 namespace: kserve-ci-e2e-test resourceVersion: "13483" uid: ecc8cc25-6c6d-4aa9-baed-199b10d97109 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:15:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:15:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:03Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a960edfca4f9df namespace: kserve-ci-e2e-test resourceVersion: "13484" uid: 0c487a33-ca05-46de-a2dd-29a0d2088742 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:15:03Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:d8f23d76f2d9886b9b5bc998ebdde5dc6c174340fd697aa08b5f7fbfe1b78fbb" metadata: creationTimestamp: "2026-04-24T19:15:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:03Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a960edfcb2f803 namespace: kserve-ci-e2e-test resourceVersion: "13485" uid: bdc083b8-d912-45ca-a06c-36cdcc371568 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:15:06Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:d8f23d76f2d9886b9b5bc998ebdde5dc6c174340fd697aa08b5f7fbfe1b78fbb" in 3.329s (3.329s including waiting). Image size: 238035010 bytes.' metadata: creationTimestamp: "2026-04-24T19:15:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:06Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a960eec329f7f9 namespace: kserve-ci-e2e-test resourceVersion: "13510" uid: 6d313d9f-5d18-4649-a6e1-91acc0dcdaf3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:15:06Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-24T19:15:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:06Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a960eec6a687d2 namespace: kserve-ci-e2e-test resourceVersion: "13511" uid: 864fd2cc-d8d2-486a-9887-fe65645bbbf4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:15:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:15:06Z" message: Started container agent metadata: creationTimestamp: "2026-04-24T19:15:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:15:06Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a960eec720b2a7 namespace: kserve-ci-e2e-test resourceVersion: "13512" uid: 41e40873-35a9-4849-a632-0fdc4bcf7b46 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T19:15:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:16:40Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:15:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:40Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a960ef37a0120c namespace: kserve-ci-e2e-test resourceVersion: "14336" uid: 15577263-11ee-4b68-9357-f0212576555f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T19:15:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:16:40Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T19:15:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:40Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a960ef37aafafd namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 7f353634-44e9-4382-891b-c0b97017fb20 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:16:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:24Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a96100f425f288 namespace: kserve-ci-e2e-test resourceVersion: "14114" uid: 06046eab-9bb2-4dd2-b2ea-5d9a3a20daba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:16:24Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-24T19:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:24Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a96100f42621a4 namespace: kserve-ci-e2e-test resourceVersion: "14116" uid: 327981a4-a355-4efe-ac0e-5758455dab51 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:16:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:24Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a96100f4265dd9 namespace: kserve-ci-e2e-test resourceVersion: "14117" uid: 0fe5e4de-c91e-4fd7-8159-a68bdd5eeeb0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w namespace: kserve-ci-e2e-test resourceVersion: "13282" uid: bc7a9a4e-23ea-4bc8-a30e-7d1f81a0d82a kind: Event lastTimestamp: "2026-04-24T19:16:48Z" message: 'Readiness probe failed: Get "https://10.132.0.38:8643/healthz": dial tcp 10.132.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:48Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w.18a96101d7f23801 namespace: kserve-ci-e2e-test resourceVersion: "14397" uid: 261157a3-650c-4ca7-942c-30fb4ad1519e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654 namespace: kserve-ci-e2e-test resourceVersion: "13267" uid: 894ba0f7-522d-4e7e-8cf9-61f345a29ee1 kind: Event lastTimestamp: "2026-04-24T19:14:39Z" message: 'Created pod: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654-2mx4w' metadata: creationTimestamp: "2026-04-24T19:14:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:14:39Z" name: isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654.18a960e87fc815d6 namespace: kserve-ci-e2e-test resourceVersion: "13281" uid: dee42e1b-861d-4019-936b-3d819344188e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-56c17-predictor namespace: kserve-ci-e2e-test resourceVersion: "13266" uid: dd9a762c-52d4-45fe-a018-0794f05d0a84 kind: Event lastTimestamp: "2026-04-24T19:14:39Z" message: Scaled up replica set isvc-raw-sklearn-batcher-56c17-predictor-6548fdf654 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:14:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:14:39Z" name: isvc-raw-sklearn-batcher-56c17-predictor.18a960e877da877f namespace: kserve-ci-e2e-test resourceVersion: "13268" uid: d55a4b71-3d4c-43a3-89e8-f6a56a90cae3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T19:14:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-56c17-predictor namespace: kserve-ci-e2e-test resourceVersion: "13275" uid: 184c6278-d68a-4873-ba60-129faf18d5b2 kind: Event lastTimestamp: "2026-04-24T19:15:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:14:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:39Z" name: isvc-raw-sklearn-batcher-56c17-predictor.18a960ebfe166028 namespace: kserve-ci-e2e-test resourceVersion: "13752" uid: 157c6c9e-bc11-463e-ae3c-0bf4d77223af reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T19:14:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-56c17-predictor namespace: kserve-ci-e2e-test resourceVersion: "13275" uid: 184c6278-d68a-4873-ba60-129faf18d5b2 kind: Event lastTimestamp: "2026-04-24T19:15:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:14:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:15:39Z" name: isvc-raw-sklearn-batcher-56c17-predictor.18a960ebfe8dc0b9 namespace: kserve-ci-e2e-test resourceVersion: "13753" uid: 4ea5b0eb-cf60-4c03-9602-e083d50dc032 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:15:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-56c17-predictor namespace: kserve-ci-e2e-test resourceVersion: "13422" uid: 184c6278-d68a-4873-ba60-129faf18d5b2 kind: Event lastTimestamp: "2026-04-24T19:16:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:15:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:16:09Z" name: isvc-raw-sklearn-batcher-56c17-predictor.18a960f9fb525a12 namespace: kserve-ci-e2e-test resourceVersion: "13967" uid: 9c82a5d9-73b8-4c83-b969-56a87e74a63d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:15:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-56c17-predictor namespace: kserve-ci-e2e-test resourceVersion: "13422" uid: 184c6278-d68a-4873-ba60-129faf18d5b2 kind: Event lastTimestamp: "2026-04-24T19:16:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:15:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:16:09Z" name: isvc-raw-sklearn-batcher-56c17-predictor.18a960f9fbbe8915 namespace: kserve-ci-e2e-test resourceVersion: "13968" uid: 85501554-4938-41aa-a645-4130aaac8d48 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T19:14:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-56c17 namespace: kserve-ci-e2e-test resourceVersion: "13257" uid: dd32cab7-eb1b-4b3c-87b5-789a31ead33f kind: Event lastTimestamp: "2026-04-24T19:16:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:14:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:16:10Z" name: isvc-raw-sklearn-batcher-56c17.18a960e87df47297 namespace: kserve-ci-e2e-test resourceVersion: "13985" uid: ef1d7b6b-1ecb-490c-ae9d-e269abfebd2f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-56c17 namespace: kserve-ci-e2e-test resourceVersion: "13257" uid: dd32cab7-eb1b-4b3c-87b5-789a31ead33f kind: Event lastTimestamp: "2026-04-24T19:14:39Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-56c17": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-56c17": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:14:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:14:39Z" name: isvc-raw-sklearn-batcher-56c17.18a960e87e3d2254 namespace: kserve-ci-e2e-test resourceVersion: "13277" uid: fc17b0b9-42db-43a4-819c-2caf110f474c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:14:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-56c17 namespace: kserve-ci-e2e-test resourceVersion: "13257" uid: dd32cab7-eb1b-4b3c-87b5-789a31ead33f kind: Event lastTimestamp: "2026-04-24T19:14:39Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-56c17": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:14:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:14:39Z" name: isvc-raw-sklearn-batcher-56c17.18a960e87e3d7dae namespace: kserve-ci-e2e-test resourceVersion: "13278" uid: c12d8d9e-e249-44c1-9401-8110a198a70a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-56c17 namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: dd32cab7-eb1b-4b3c-87b5-789a31ead33f kind: Event lastTimestamp: "2026-04-24T19:16:10Z" message: InferenceService [isvc-raw-sklearn-batcher-56c17] is Ready metadata: creationTimestamp: "2026-04-24T19:16:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:16:10Z" name: isvc-raw-sklearn-batcher-56c17.18a960fdad5a1ffa namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 0eba253a-899b-4d1f-a828-fd4f7fa25f52 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:25:31.081832Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-b4ed0b-predictor-58497c968-9lk85 namespace: kserve-ci-e2e-test resourceVersion: "18899" uid: 633f22e8-63a8-478d-bd5f-44155c31cdc7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-b4ed0b-predictor-58497c968-9lk85 to ip-10-0-138-52.ec2.internal metadata: creationTimestamp: "2026-04-24T19:25:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:25:31Z" name: isvc-secondary-b4ed0b-predictor-58497c968-9lk85.18a961802b23b80f namespace: kserve-ci-e2e-test resourceVersion: "18903" uid: cbf6c575-aadd-4bb0-8a69-57bf0b77e487 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df6b44bbb-hm2bm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-b4ed0b-predictor-58497c968-9lk85 namespace: kserve-ci-e2e-test resourceVersion: "18907" uid: 633f22e8-63a8-478d-bd5f-44155c31cdc7 kind: Event lastTimestamp: "2026-04-24T19:25:31Z" message: Add eth0 [10.132.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:25:31Z" name: isvc-secondary-b4ed0b-predictor-58497c968-9lk85.18a9618044fe9c52 namespace: kserve-ci-e2e-test resourceVersion: "18917" uid: 80e0d534-d2f3-43ac-80f3-45ee50d752cc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-b4ed0b-predictor-58497c968-9lk85 namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: 633f22e8-63a8-478d-bd5f-44155c31cdc7 kind: Event lastTimestamp: "2026-04-24T19:25:35Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0183e14c0f4ec83676f700a2c092ab4e3599fa115c6cab4546b23e5207166882" already present on machine metadata: creationTimestamp: "2026-04-24T19:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:35Z" name: isvc-secondary-b4ed0b-predictor-58497c968-9lk85.18a961804673874e namespace: kserve-ci-e2e-test resourceVersion: "18963" uid: a65cec60-072c-40e2-b262-53812ee1aa2e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-b4ed0b-predictor-58497c968-9lk85 namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: 633f22e8-63a8-478d-bd5f-44155c31cdc7 kind: Event lastTimestamp: "2026-04-24T19:25:35Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T19:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:35Z" name: isvc-secondary-b4ed0b-predictor-58497c968-9lk85.18a961804959c913 namespace: kserve-ci-e2e-test resourceVersion: "18967" uid: 7bcb201b-d5da-41f8-aa42-15aad39998d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:25:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-b4ed0b-predictor-58497c968-9lk85 namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: 633f22e8-63a8-478d-bd5f-44155c31cdc7 kind: Event lastTimestamp: "2026-04-24T19:25:35Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T19:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:35Z" name: isvc-secondary-b4ed0b-predictor-58497c968-9lk85.18a9618049d0f024 namespace: kserve-ci-e2e-test resourceVersion: "18968" uid: 15b09dd3-54ac-4ef6-94e6-0553c6c4db5f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-b4ed0b-predictor-58497c968-9lk85 namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: 633f22e8-63a8-478d-bd5f-44155c31cdc7 kind: Event lastTimestamp: "2026-04-24T19:25:42Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-b4ed0b-predictor-58497c968-9lk85_kserve-ci-e2e-test(633f22e8-63a8-478d-bd5f-44155c31cdc7) metadata: creationTimestamp: "2026-04-24T19:25:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:42Z" name: isvc-secondary-b4ed0b-predictor-58497c968-9lk85.18a96182ef71f829 namespace: kserve-ci-e2e-test resourceVersion: "19024" uid: 422c12e6-e2b1-4de7-a815-d62c517856bb reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-b4ed0b-predictor-58497c968 namespace: kserve-ci-e2e-test resourceVersion: "18890" uid: f145b2db-742d-40e3-bd52-ef221e0635a2 kind: Event lastTimestamp: "2026-04-24T19:25:31Z" message: 'Created pod: isvc-secondary-b4ed0b-predictor-58497c968-9lk85' metadata: creationTimestamp: "2026-04-24T19:25:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:31Z" name: isvc-secondary-b4ed0b-predictor-58497c968.18a961802ab8138f namespace: kserve-ci-e2e-test resourceVersion: "18900" uid: a03dd87f-530d-499b-b162-df544e5322db reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-b4ed0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18888" uid: 9040c849-7fd5-4306-bc42-3b6526324b14 kind: Event lastTimestamp: "2026-04-24T19:25:30Z" message: Scaled up replica set isvc-secondary-b4ed0b-predictor-58497c968 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:25:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:30Z" name: isvc-secondary-b4ed0b-predictor.18a9618024fe99fc namespace: kserve-ci-e2e-test resourceVersion: "18892" uid: d395767e-fd91-4e8e-a4e9-21b61d1c1585 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-b4ed0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18889" uid: dae6f8bb-0f2f-497d-9a1c-032504e8bdae kind: Event lastTimestamp: "2026-04-24T19:25:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:25:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:45Z" name: isvc-secondary-b4ed0b-predictor.18a96183a32cfb55 namespace: kserve-ci-e2e-test resourceVersion: "19048" uid: 016900e9-cc11-4d42-a83d-f7a64274a78e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-b4ed0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18889" uid: dae6f8bb-0f2f-497d-9a1c-032504e8bdae kind: Event lastTimestamp: "2026-04-24T19:25:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:25:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:45Z" name: isvc-secondary-b4ed0b-predictor.18a96183a39b2fa9 namespace: kserve-ci-e2e-test resourceVersion: "19050" uid: cc47b2fe-a7e7-431c-a961-d63bad48979a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T19:25:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-b4ed0b namespace: kserve-ci-e2e-test resourceVersion: "18879" uid: c1c8c4d5-0c36-40ae-9c73-0ad54369fbd6 kind: Event lastTimestamp: "2026-04-24T19:25:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:25:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:25:43Z" name: isvc-secondary-b4ed0b.18a96180248eb5e5 namespace: kserve-ci-e2e-test resourceVersion: "19029" uid: 49e1aa93-a6b3-43d8-87f6-8afb6d5df069 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:25:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-b4ed0b namespace: kserve-ci-e2e-test resourceVersion: "18879" uid: c1c8c4d5-0c36-40ae-9c73-0ad54369fbd6 kind: Event lastTimestamp: "2026-04-24T19:25:31Z" message: 'Failed to update status for InferenceService "isvc-secondary-b4ed0b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-b4ed0b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:25:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:25:31Z" name: isvc-secondary-b4ed0b.18a9618025554522 namespace: kserve-ci-e2e-test resourceVersion: "18909" uid: 01aa4382-b1c1-406a-8d95-c572be3a8946 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:25:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-b4ed0b namespace: kserve-ci-e2e-test resourceVersion: "18879" uid: c1c8c4d5-0c36-40ae-9c73-0ad54369fbd6 kind: Event lastTimestamp: "2026-04-24T19:25:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-b4ed0b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:25:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:25:31Z" name: isvc-secondary-b4ed0b.18a961802555a72f namespace: kserve-ci-e2e-test resourceVersion: "18910" uid: 5780dfb7-841d-4078-947d-946e7c82ccc1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:16:24.761928Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k namespace: kserve-ci-e2e-test resourceVersion: "14128" uid: 87de199e-12a0-4a99-a905-77533d44ca3b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k to ip-10-0-138-52.ec2.internal metadata: creationTimestamp: "2026-04-24T19:16:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:16:24Z" name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k.18a96100f7ef8a3b namespace: kserve-ci-e2e-test resourceVersion: "14138" uid: 3c37a6ae-f978-4069-bc97-c7f76ef1f4c4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df6b44bbb-hm2bm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:25Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k namespace: kserve-ci-e2e-test resourceVersion: "14144" uid: 87de199e-12a0-4a99-a905-77533d44ca3b kind: Event lastTimestamp: "2026-04-24T19:16:25Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:16:25Z" name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k.18a9610111eea6da namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: b7f5ee4b-83b6-45ee-b3a1-5e5e5a9f3e5f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k namespace: kserve-ci-e2e-test resourceVersion: "14133" uid: 87de199e-12a0-4a99-a905-77533d44ca3b kind: Event lastTimestamp: "2026-04-24T19:16:25Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0183e14c0f4ec83676f700a2c092ab4e3599fa115c6cab4546b23e5207166882" already present on machine metadata: creationTimestamp: "2026-04-24T19:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:25Z" name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k.18a9610113642104 namespace: kserve-ci-e2e-test resourceVersion: "14173" uid: 9eab94a5-b0d8-4c55-bce4-948fceb4a4a0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k namespace: kserve-ci-e2e-test resourceVersion: "14133" uid: 87de199e-12a0-4a99-a905-77533d44ca3b kind: Event lastTimestamp: "2026-04-24T19:16:25Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T19:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:25Z" name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k.18a96101173c3831 namespace: kserve-ci-e2e-test resourceVersion: "14176" uid: a63e8e3d-6ea6-4e84-8587-2dbbc1882d2e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k namespace: kserve-ci-e2e-test resourceVersion: "14133" uid: 87de199e-12a0-4a99-a905-77533d44ca3b kind: Event lastTimestamp: "2026-04-24T19:16:25Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T19:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:25Z" name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k.18a9610117bd6d0a namespace: kserve-ci-e2e-test resourceVersion: "14178" uid: 8eeb29f4-5101-4ef2-a614-6acd742bb03a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k namespace: kserve-ci-e2e-test resourceVersion: "14133" uid: 87de199e-12a0-4a99-a905-77533d44ca3b kind: Event lastTimestamp: "2026-04-24T19:16:29Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:16:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:29Z" name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k.18a96102256bd32e namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 51701525-2807-4341-bbf1-6c8ed29eee56 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k namespace: kserve-ci-e2e-test resourceVersion: "14133" uid: 87de199e-12a0-4a99-a905-77533d44ca3b kind: Event lastTimestamp: "2026-04-24T19:16:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:16:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:29Z" name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k.18a96102299776f7 namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: 42e506f2-84bc-4d92-8266-0c6e4cbd806d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k namespace: kserve-ci-e2e-test resourceVersion: "14133" uid: 87de199e-12a0-4a99-a905-77533d44ca3b kind: Event lastTimestamp: "2026-04-24T19:16:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:16:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:29Z" name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k.18a961022a1ced4a namespace: kserve-ci-e2e-test resourceVersion: "14237" uid: da0d5e42-d358-4369-b1aa-cc2d4f6093e9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k namespace: kserve-ci-e2e-test resourceVersion: "14133" uid: 87de199e-12a0-4a99-a905-77533d44ca3b kind: Event lastTimestamp: "2026-04-24T19:16:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:16:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:29Z" name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k.18a961022a2a9481 namespace: kserve-ci-e2e-test resourceVersion: "14238" uid: 859c4e6a-3119-4450-87d8-4a8c02880e28 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k namespace: kserve-ci-e2e-test resourceVersion: "14133" uid: 87de199e-12a0-4a99-a905-77533d44ca3b kind: Event lastTimestamp: "2026-04-24T19:16:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:16:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:29Z" name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k.18a961022db0835f namespace: kserve-ci-e2e-test resourceVersion: "14242" uid: 5e235170-8044-4a3c-8c67-8e7c1a0dffaf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k namespace: kserve-ci-e2e-test resourceVersion: "14133" uid: 87de199e-12a0-4a99-a905-77533d44ca3b kind: Event lastTimestamp: "2026-04-24T19:16:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:16:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:29Z" name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k.18a961022e29ccce namespace: kserve-ci-e2e-test resourceVersion: "14244" uid: 24869da1-e6f1-46d8-8ce6-809685b88a7e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T19:16:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k namespace: kserve-ci-e2e-test resourceVersion: "14133" uid: 87de199e-12a0-4a99-a905-77533d44ca3b kind: Event lastTimestamp: "2026-04-24T19:18:16Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:16:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:16Z" name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k.18a9610261c2b6fb namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: a73fb559-be7c-4334-9d2d-624b77f8f09e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k namespace: kserve-ci-e2e-test resourceVersion: "14133" uid: 87de199e-12a0-4a99-a905-77533d44ca3b kind: Event lastTimestamp: "2026-04-24T19:18:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:15Z" name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k.18a9611aa377719d namespace: kserve-ci-e2e-test resourceVersion: "15155" uid: 99482cf9-d097-41af-8853-9870abcdb4a4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k namespace: kserve-ci-e2e-test resourceVersion: "14133" uid: 87de199e-12a0-4a99-a905-77533d44ca3b kind: Event lastTimestamp: "2026-04-24T19:18:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:15Z" name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k.18a9611aa37807b6 namespace: kserve-ci-e2e-test resourceVersion: "15157" uid: db45ffec-94ff-4960-a2d7-d695ea151e71 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k namespace: kserve-ci-e2e-test resourceVersion: "14133" uid: 87de199e-12a0-4a99-a905-77533d44ca3b kind: Event lastTimestamp: "2026-04-24T19:18:16Z" message: 'Readiness probe failed: Get "https://10.132.0.39:8643/healthz": dial tcp 10.132.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:18:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:16Z" name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k.18a9611b100cb4f6 namespace: kserve-ci-e2e-test resourceVersion: "15252" uid: f816f651-3794-4a8d-a9a1-7cdacf31465c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8 namespace: kserve-ci-e2e-test resourceVersion: "14107" uid: 65473e35-611e-4d91-9e29-fb9614fa4ca3 kind: Event lastTimestamp: "2026-04-24T19:16:24Z" message: 'Created pod: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8-x9j4k' metadata: creationTimestamp: "2026-04-24T19:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:16:24Z" name: isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8.18a96100f789b744 namespace: kserve-ci-e2e-test resourceVersion: "14132" uid: 7ae55680-3b97-4db0-8a99-dfeb2cd24969 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-1c30c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14098" uid: 058cad19-6aac-4ed0-9869-d37818869b68 kind: Event lastTimestamp: "2026-04-24T19:16:24Z" message: Scaled up replica set isvc-sklearn-graph-raw-1c30c-predictor-bcf44fbc8 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:16:24Z" name: isvc-sklearn-graph-raw-1c30c-predictor.18a96100f307a010 namespace: kserve-ci-e2e-test resourceVersion: "14109" uid: 83ccdd63-8857-4c8c-8216-7673c3d49e75 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:16:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-1c30c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14104" uid: ce301454-d025-4824-a62e-d01c6bd4076f kind: Event lastTimestamp: "2026-04-24T19:17:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:16:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:17:09Z" name: isvc-sklearn-graph-raw-1c30c-predictor.18a96104717b07c0 namespace: kserve-ci-e2e-test resourceVersion: "14565" uid: 8fad17c2-02bd-4238-84b8-071629a7bf3e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:16:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-1c30c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14104" uid: ce301454-d025-4824-a62e-d01c6bd4076f kind: Event lastTimestamp: "2026-04-24T19:17:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:16:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:17:09Z" name: isvc-sklearn-graph-raw-1c30c-predictor.18a9610471e42a3a namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: 4a673608-2446-4bb2-8260-d9db02981ede reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T19:17:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-1c30c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14325" uid: ce301454-d025-4824-a62e-d01c6bd4076f kind: Event lastTimestamp: "2026-04-24T19:18:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:17:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:18:09Z" name: isvc-sklearn-graph-raw-1c30c-predictor.18a9610eefe5e5f2 namespace: kserve-ci-e2e-test resourceVersion: "15053" uid: a6d54fcc-186c-4119-80bc-f66af5b77906 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T19:17:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-1c30c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14325" uid: ce301454-d025-4824-a62e-d01c6bd4076f kind: Event lastTimestamp: "2026-04-24T19:18:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:17:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:18:09Z" name: isvc-sklearn-graph-raw-1c30c-predictor.18a9610ef0605707 namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: 16df494a-0e0c-4cc9-8b17-0702ffcf72de reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T19:16:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-1c30c namespace: kserve-ci-e2e-test resourceVersion: "14079" uid: c5e54565-513d-45f8-b694-d0cab9ad8acb kind: Event lastTimestamp: "2026-04-24T19:17:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:17:46Z" name: isvc-sklearn-graph-raw-1c30c.18a96100f2bb39c9 namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: d8fc68cf-3196-42dc-a927-b5d52818e2b9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-1c30c namespace: kserve-ci-e2e-test resourceVersion: "14079" uid: c5e54565-513d-45f8-b694-d0cab9ad8acb kind: Event lastTimestamp: "2026-04-24T19:16:24Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-1c30c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-1c30c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:16:24Z" name: isvc-sklearn-graph-raw-1c30c.18a96100f334a46d namespace: kserve-ci-e2e-test resourceVersion: "14111" uid: 1f5cfae0-cd5a-4e61-8d06-727126cd7338 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:17:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-1c30c namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: c5e54565-513d-45f8-b694-d0cab9ad8acb kind: Event lastTimestamp: "2026-04-24T19:17:46Z" message: InferenceService [isvc-sklearn-graph-raw-1c30c] is Ready metadata: creationTimestamp: "2026-04-24T19:17:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:17:46Z" name: isvc-sklearn-graph-raw-1c30c.18a9611418ac8985 namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: 37b57e9b-bf4f-460a-8272-98e7084674dd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:18:15.073990Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 7ca8ad97-5405-4afe-8fc6-c4a1f0c545c9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f to ip-10-0-138-52.ec2.internal metadata: creationTimestamp: "2026-04-24T19:18:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:18:15Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f.18a9611aa70c1505 namespace: kserve-ci-e2e-test resourceVersion: "15177" uid: 873f56ca-c31e-468a-93b5-b216de33f1e2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df6b44bbb-hm2bm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:15Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f namespace: kserve-ci-e2e-test resourceVersion: "15194" uid: 7ca8ad97-5405-4afe-8fc6-c4a1f0c545c9 kind: Event lastTimestamp: "2026-04-24T19:18:15Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:18:15Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f.18a9611ac10edeeb namespace: kserve-ci-e2e-test resourceVersion: "15224" uid: 4007986a-8cbf-4cc0-aa64-71229974505e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: 7ca8ad97-5405-4afe-8fc6-c4a1f0c545c9 kind: Event lastTimestamp: "2026-04-24T19:18:15Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0183e14c0f4ec83676f700a2c092ab4e3599fa115c6cab4546b23e5207166882" already present on machine metadata: creationTimestamp: "2026-04-24T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:15Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f.18a9611ac27d3af6 namespace: kserve-ci-e2e-test resourceVersion: "15226" uid: b3eb6c38-41ca-4864-8488-98a27003a8fc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: 7ca8ad97-5405-4afe-8fc6-c4a1f0c545c9 kind: Event lastTimestamp: "2026-04-24T19:18:15Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:15Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f.18a9611ac545454d namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: 6528563e-b6b3-41aa-8477-a1aaabe78929 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: 7ca8ad97-5405-4afe-8fc6-c4a1f0c545c9 kind: Event lastTimestamp: "2026-04-24T19:18:15Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:15Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f.18a9611ac5ba3d43 namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: 0eb21abb-c535-4637-8c49-7bacd24bd673 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: 7ca8ad97-5405-4afe-8fc6-c4a1f0c545c9 kind: Event lastTimestamp: "2026-04-24T19:18:20Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:20Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f.18a9611bdd33a0cc namespace: kserve-ci-e2e-test resourceVersion: "15293" uid: 3bec7ceb-6c02-49f4-9739-f1c81fb93648 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: 7ca8ad97-5405-4afe-8fc6-c4a1f0c545c9 kind: Event lastTimestamp: "2026-04-24T19:18:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:20Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f.18a9611be56ebd23 namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 562347ec-7c36-4621-9dbf-39c373e72c4d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: 7ca8ad97-5405-4afe-8fc6-c4a1f0c545c9 kind: Event lastTimestamp: "2026-04-24T19:18:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:20Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f.18a9611be5e5bfe5 namespace: kserve-ci-e2e-test resourceVersion: "15303" uid: 1d24478f-1061-4e83-a5ac-aa7fdc2e3c59 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: 7ca8ad97-5405-4afe-8fc6-c4a1f0c545c9 kind: Event lastTimestamp: "2026-04-24T19:18:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:20Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f.18a9611be5f3a57c namespace: kserve-ci-e2e-test resourceVersion: "15304" uid: ca300bed-7c62-4faa-891f-902d6a7549eb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: 7ca8ad97-5405-4afe-8fc6-c4a1f0c545c9 kind: Event lastTimestamp: "2026-04-24T19:18:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:20Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f.18a9611be9cb3237 namespace: kserve-ci-e2e-test resourceVersion: "15306" uid: 3d15637e-58cc-43bb-9488-c0f6de2ecf68 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: 7ca8ad97-5405-4afe-8fc6-c4a1f0c545c9 kind: Event lastTimestamp: "2026-04-24T19:18:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:18:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:20Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f.18a9611bea424c5e namespace: kserve-ci-e2e-test resourceVersion: "15307" uid: d68c8465-8059-4bf4-bc12-959286ed41d3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T19:18:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: 7ca8ad97-5405-4afe-8fc6-c4a1f0c545c9 kind: Event lastTimestamp: "2026-04-24T19:19:58Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:18:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:19:58Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f.18a9611c5546e993 namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: 78de350c-dc8c-4cbc-865e-5bd52585d051 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: 7ca8ad97-5405-4afe-8fc6-c4a1f0c545c9 kind: Event lastTimestamp: "2026-04-24T19:19:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:19:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:19:55Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f.18a96131fd043804 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: b6d1d632-bba7-4d7c-a086-59573a936f32 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: 7ca8ad97-5405-4afe-8fc6-c4a1f0c545c9 kind: Event lastTimestamp: "2026-04-24T19:19:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:19:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:19:55Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f.18a96131fd04ac6a namespace: kserve-ci-e2e-test resourceVersion: "16120" uid: 84a0e048-6c15-442c-892e-21579c296295 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: 7ca8ad97-5405-4afe-8fc6-c4a1f0c545c9 kind: Event lastTimestamp: "2026-04-24T19:19:58Z" message: 'Readiness probe failed: Get "https://10.132.0.42:8643/healthz": dial tcp 10.132.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:19:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:19:58Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f.18a96132af8cebc1 namespace: kserve-ci-e2e-test resourceVersion: "16196" uid: 589e7b74-038d-4dc0-a5a5-4e8e3e5d8373 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: 96363a94-f043-41cd-8cdc-6d5e398e2c35 kind: Event lastTimestamp: "2026-04-24T19:18:15Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd-fr92f' metadata: creationTimestamp: "2026-04-24T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:18:15Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd.18a9611aa6a60430 namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: 0e5fb4d8-b1c1-4725-960e-c086d1f575e4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-1911c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15137" uid: 89d474c0-0590-4e72-95b4-95e84276bb70 kind: Event lastTimestamp: "2026-04-24T19:18:14Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-1911c-predictor-5f544595fd from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:18:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:18:14Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor.18a9611aa261e4a9 namespace: kserve-ci-e2e-test resourceVersion: "15146" uid: 9a0d08b0-51d7-450f-8ab0-787653e80c69 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:18:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-1911c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 619430b2-e3bd-40c4-b32e-5e4698e53e0f kind: Event lastTimestamp: "2026-04-24T19:18:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:18:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:18:45Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor.18a9611e21a10506 namespace: kserve-ci-e2e-test resourceVersion: "15508" uid: ce97476a-9200-49aa-a94e-d4e5c063a921 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:18:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-1911c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 619430b2-e3bd-40c4-b32e-5e4698e53e0f kind: Event lastTimestamp: "2026-04-24T19:18:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:18:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:18:45Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor.18a9611e220d9206 namespace: kserve-ci-e2e-test resourceVersion: "15509" uid: 40edac2e-1ec7-4d2e-b72c-7b3ad78aa2b3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T19:19:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-1911c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15395" uid: 619430b2-e3bd-40c4-b32e-5e4698e53e0f kind: Event lastTimestamp: "2026-04-24T19:19:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:19:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:19:45Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor.18a961252007aaeb namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 3b067f2e-3a6a-48ef-8325-0c03b86f597f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T19:19:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-1911c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15395" uid: 619430b2-e3bd-40c4-b32e-5e4698e53e0f kind: Event lastTimestamp: "2026-04-24T19:19:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:19:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:19:45Z" name: isvc-sklearn-graph-raw-hpa-1911c-predictor.18a9612520728e97 namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: f8d831a1-6451-41bd-9281-eac20586e990 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T19:18:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-1911c namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: 16151e96-3cbb-40eb-ac4a-020efcc2eb25 kind: Event lastTimestamp: "2026-04-24T19:19:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:18:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:19:28Z" name: isvc-sklearn-graph-raw-hpa-1911c.18a9611aa2224abc namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: afd0f519-ce2a-4d8b-ba0c-4d270346e3f0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-1911c namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: 16151e96-3cbb-40eb-ac4a-020efcc2eb25 kind: Event lastTimestamp: "2026-04-24T19:18:14Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-1911c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-1911c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:18:15Z" name: isvc-sklearn-graph-raw-hpa-1911c.18a9611aa2939ccc namespace: kserve-ci-e2e-test resourceVersion: "15148" uid: 04466635-003e-42f2-badd-65650a902a5d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-1911c namespace: kserve-ci-e2e-test resourceVersion: "15831" uid: 16151e96-3cbb-40eb-ac4a-020efcc2eb25 kind: Event lastTimestamp: "2026-04-24T19:19:28Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-1911c] is Ready metadata: creationTimestamp: "2026-04-24T19:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:19:28Z" name: isvc-sklearn-graph-raw-hpa-1911c.18a9612bbaae8d63 namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: d4d201ae-e2f0-4422-83ad-57db4aa6c87e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:21:40.627868Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm namespace: kserve-ci-e2e-test resourceVersion: "17073" uid: 8c51954e-999a-4851-9c45-16f29af69591 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm to ip-10-0-138-52.ec2.internal metadata: creationTimestamp: "2026-04-24T19:21:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:21:40Z" name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm.18a9614a83034b1d namespace: kserve-ci-e2e-test resourceVersion: "17078" uid: ed901a26-c3e9-4920-b5b6-1205c19fe120 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df6b44bbb-hm2bm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:21:40Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: 8c51954e-999a-4851-9c45-16f29af69591 kind: Event lastTimestamp: "2026-04-24T19:21:40Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-dd3f9-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T19:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:21:40Z" name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm.18a9614a90ac7850 namespace: kserve-ci-e2e-test resourceVersion: "17103" uid: bbaf590b-7e4a-48ba-b424-774adb52fef6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:21:41Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm namespace: kserve-ci-e2e-test resourceVersion: "17083" uid: 8c51954e-999a-4851-9c45-16f29af69591 kind: Event lastTimestamp: "2026-04-24T19:21:41Z" message: Add eth0 [10.132.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:21:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:21:41Z" name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm.18a9614ac10150e9 namespace: kserve-ci-e2e-test resourceVersion: "17116" uid: 997b5d55-c6eb-46ae-aa32-37b2e45700e4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:21:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: 8c51954e-999a-4851-9c45-16f29af69591 kind: Event lastTimestamp: "2026-04-24T19:21:41Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0183e14c0f4ec83676f700a2c092ab4e3599fa115c6cab4546b23e5207166882" already present on machine metadata: creationTimestamp: "2026-04-24T19:21:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:21:41Z" name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm.18a9614ac2742c11 namespace: kserve-ci-e2e-test resourceVersion: "17119" uid: 28141c92-e60b-4963-befe-ad5d6b537111 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:21:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: 8c51954e-999a-4851-9c45-16f29af69591 kind: Event lastTimestamp: "2026-04-24T19:21:41Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T19:21:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:21:41Z" name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm.18a9614ac60443e0 namespace: kserve-ci-e2e-test resourceVersion: "17121" uid: 92c6132d-986d-4ff0-ab47-ac5c6ba32456 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:21:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: 8c51954e-999a-4851-9c45-16f29af69591 kind: Event lastTimestamp: "2026-04-24T19:21:41Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T19:21:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:21:41Z" name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm.18a9614ac67c4227 namespace: kserve-ci-e2e-test resourceVersion: "17122" uid: 956c8b77-9fc4-4b10-ba4a-1a1a84439af6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: 8c51954e-999a-4851-9c45-16f29af69591 kind: Event lastTimestamp: "2026-04-24T19:21:46Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:21:46Z" name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm.18a9614bc7e7585c namespace: kserve-ci-e2e-test resourceVersion: "17159" uid: 4c2796c3-a54c-4a20-8537-9ef472abfcd3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: 8c51954e-999a-4851-9c45-16f29af69591 kind: Event lastTimestamp: "2026-04-24T19:21:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:21:46Z" name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm.18a9614bcbabb502 namespace: kserve-ci-e2e-test resourceVersion: "17164" uid: b3929c87-b0e3-4228-8f7e-7b980a11eb7b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: 8c51954e-999a-4851-9c45-16f29af69591 kind: Event lastTimestamp: "2026-04-24T19:21:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:21:46Z" name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm.18a9614bcc3b0bcd namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: ccabbd9e-b020-4f33-b9cc-41c62c64b7a1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: 8c51954e-999a-4851-9c45-16f29af69591 kind: Event lastTimestamp: "2026-04-24T19:21:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:21:46Z" name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm.18a9614bcc48bcf6 namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: 0f9a8057-482f-4294-b8bb-2d57aed6bb9d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: 8c51954e-999a-4851-9c45-16f29af69591 kind: Event lastTimestamp: "2026-04-24T19:21:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:21:46Z" name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm.18a9614bd088b4d6 namespace: kserve-ci-e2e-test resourceVersion: "17169" uid: 9c4c52f7-00f6-4b37-bf19-4eb2cd4112d9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:21:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: 8c51954e-999a-4851-9c45-16f29af69591 kind: Event lastTimestamp: "2026-04-24T19:21:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:21:46Z" name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm.18a9614bd0fea74e namespace: kserve-ci-e2e-test resourceVersion: "17171" uid: f0c45aaf-5394-4bcf-a990-d88fb2e5c700 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 17 eventTime: null firstTimestamp: "2026-04-24T19:21:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: 8c51954e-999a-4851-9c45-16f29af69591 kind: Event lastTimestamp: "2026-04-24T19:24:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:21:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:13Z" name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm.18a9614c03e2c6bb namespace: kserve-ci-e2e-test resourceVersion: "18301" uid: fe0d5b0f-f5e8-404c-9f13-09bcda9412b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: 8c51954e-999a-4851-9c45-16f29af69591 kind: Event lastTimestamp: "2026-04-24T19:24:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:10Z" name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm.18a9616d7ac3d691 namespace: kserve-ci-e2e-test resourceVersion: "18244" uid: 26bb4cfa-01d1-442d-bc25-8c33d1476dd5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: 8c51954e-999a-4851-9c45-16f29af69591 kind: Event lastTimestamp: "2026-04-24T19:24:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:10Z" name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm.18a9616d7ac43a8b namespace: kserve-ci-e2e-test resourceVersion: "18247" uid: 23835a31-1692-4a77-918a-5dcdaf963921 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:24:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: 8c51954e-999a-4851-9c45-16f29af69591 kind: Event lastTimestamp: "2026-04-24T19:24:18Z" message: 'Readiness probe failed: Get "https://10.132.0.46:8643/healthz": dial tcp 10.132.0.46:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:24:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:24:18Z" name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm.18a9616e02563507 namespace: kserve-ci-e2e-test resourceVersion: "18344" uid: d4216d91-74cc-450d-8c1c-a18984c9e974 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:21:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68 namespace: kserve-ci-e2e-test resourceVersion: "17049" uid: 0ef3c15b-34f3-4c17-89b5-809cf5e1513c kind: Event lastTimestamp: "2026-04-24T19:21:40Z" message: 'Created pod: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68-4rdgm' metadata: creationTimestamp: "2026-04-24T19:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:21:40Z" name: isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68.18a9614a82999390 namespace: kserve-ci-e2e-test resourceVersion: "17074" uid: 1efee257-886c-4c2d-a851-9dd8b272f483 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:21:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-dd3f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "17044" uid: 174ceccd-34a3-4b53-b4ef-ff7396d8da58 kind: Event lastTimestamp: "2026-04-24T19:21:40Z" message: Scaled up replica set isvc-sklearn-scale-raw-dd3f9-predictor-7c6474bc68 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:21:40Z" name: isvc-sklearn-scale-raw-dd3f9-predictor.18a9614a7f4d596f namespace: kserve-ci-e2e-test resourceVersion: "17055" uid: c9e4d9a2-9bfa-49b9-ba02-fc731cf06c2f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:21:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-dd3f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "17048" uid: 75dd26f7-474b-456f-a04e-34787b0b4dda kind: Event lastTimestamp: "2026-04-24T19:22:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:21:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:22:25Z" name: isvc-sklearn-scale-raw-dd3f9-predictor.18a9614dfe5585e9 namespace: kserve-ci-e2e-test resourceVersion: "17460" uid: dbb41d1a-763d-4e14-9c55-d4ace5e1a3d5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:21:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-dd3f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "17048" uid: 75dd26f7-474b-456f-a04e-34787b0b4dda kind: Event lastTimestamp: "2026-04-24T19:22:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:21:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:22:25Z" name: isvc-sklearn-scale-raw-dd3f9-predictor.18a9614dfeb8d325 namespace: kserve-ci-e2e-test resourceVersion: "17461" uid: 15589726-bee6-43c6-8e33-b0609e38912d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T19:22:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-dd3f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "17244" uid: 75dd26f7-474b-456f-a04e-34787b0b4dda kind: Event lastTimestamp: "2026-04-24T19:24:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:24:10Z" name: isvc-sklearn-scale-raw-dd3f9-predictor.18a961587bda524e namespace: kserve-ci-e2e-test resourceVersion: "18211" uid: ae2b65cc-5e5a-4dc3-9a02-d3ecc611ef5c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T19:22:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-dd3f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "17244" uid: 75dd26f7-474b-456f-a04e-34787b0b4dda kind: Event lastTimestamp: "2026-04-24T19:24:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:24:10Z" name: isvc-sklearn-scale-raw-dd3f9-predictor.18a961587c44ae13 namespace: kserve-ci-e2e-test resourceVersion: "18212" uid: 4314b6a7-dd6f-41ae-abf2-d4577ed17d1b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-24T19:21:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-dd3f9 namespace: kserve-ci-e2e-test resourceVersion: "17030" uid: 33c588b3-d086-4294-a651-19b02f30b51d kind: Event lastTimestamp: "2026-04-24T19:24:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:24:03Z" name: isvc-sklearn-scale-raw-dd3f9.18a9614a7f1fbd0a namespace: kserve-ci-e2e-test resourceVersion: "18163" uid: 6d2f3659-5339-43d7-8b2f-a34826b1cf3c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:21:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-dd3f9 namespace: kserve-ci-e2e-test resourceVersion: "17030" uid: 33c588b3-d086-4294-a651-19b02f30b51d kind: Event lastTimestamp: "2026-04-24T19:21:40Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-dd3f9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-dd3f9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:21:40Z" name: isvc-sklearn-scale-raw-dd3f9.18a9614a7f844234 namespace: kserve-ci-e2e-test resourceVersion: "17097" uid: 12ed3e5c-4706-4966-b02e-c8e902d09374 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:21:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-dd3f9 namespace: kserve-ci-e2e-test resourceVersion: "17045" uid: 33c588b3-d086-4294-a651-19b02f30b51d kind: Event lastTimestamp: "2026-04-24T19:21:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-dd3f9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:21:40Z" name: isvc-sklearn-scale-raw-dd3f9.18a9614a8529a7f9 namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 9acd2f1c-7bbc-4a0a-988b-d2c850b47c22 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:24:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-dd3f9 namespace: kserve-ci-e2e-test resourceVersion: "18160" uid: 33c588b3-d086-4294-a651-19b02f30b51d kind: Event lastTimestamp: "2026-04-24T19:24:03Z" message: InferenceService [isvc-sklearn-scale-raw-dd3f9] is Ready metadata: creationTimestamp: "2026-04-24T19:24:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:24:03Z" name: isvc-sklearn-scale-raw-dd3f9.18a9616bc9bafc0f namespace: kserve-ci-e2e-test resourceVersion: "18162" uid: 71667a08-e786-4839-9cb0-98d9d85b8e92 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:16:24.813765Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: f4821ffc-0241-4b44-9207-1e7f3b37cce3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c to ip-10-0-138-52.ec2.internal metadata: creationTimestamp: "2026-04-24T19:16:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:16:24Z" name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c.18a96100fb0679ca namespace: kserve-ci-e2e-test resourceVersion: "14154" uid: 1b98d588-679a-47ac-a696-369d1c9cd8bd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df6b44bbb-hm2bm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:25Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c namespace: kserve-ci-e2e-test resourceVersion: "14160" uid: f4821ffc-0241-4b44-9207-1e7f3b37cce3 kind: Event lastTimestamp: "2026-04-24T19:16:25Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:16:25Z" name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c.18a961011623f198 namespace: kserve-ci-e2e-test resourceVersion: "14174" uid: 82945abf-ae51-4faf-a7a6-c2a47012d4ff reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: f4821ffc-0241-4b44-9207-1e7f3b37cce3 kind: Event lastTimestamp: "2026-04-24T19:16:25Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0183e14c0f4ec83676f700a2c092ab4e3599fa115c6cab4546b23e5207166882" already present on machine metadata: creationTimestamp: "2026-04-24T19:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:25Z" name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c.18a9610117a78274 namespace: kserve-ci-e2e-test resourceVersion: "14177" uid: 6079c925-db17-4208-aee9-b7cab23cf22c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: f4821ffc-0241-4b44-9207-1e7f3b37cce3 kind: Event lastTimestamp: "2026-04-24T19:16:25Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T19:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:25Z" name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c.18a961011b194796 namespace: kserve-ci-e2e-test resourceVersion: "14180" uid: ad3a0de0-eacf-458a-b9d6-9243e5f279ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: f4821ffc-0241-4b44-9207-1e7f3b37cce3 kind: Event lastTimestamp: "2026-04-24T19:16:25Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T19:16:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:25Z" name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c.18a961011b95c2a9 namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: 7ec62329-7eb2-4735-9860-27063d700e79 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: f4821ffc-0241-4b44-9207-1e7f3b37cce3 kind: Event lastTimestamp: "2026-04-24T19:16:29Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-24T19:16:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:29Z" name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c.18a96102255a5238 namespace: kserve-ci-e2e-test resourceVersion: "14230" uid: cf881610-8b44-41c9-a363-35601c6e21e5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: f4821ffc-0241-4b44-9207-1e7f3b37cce3 kind: Event lastTimestamp: "2026-04-24T19:16:47Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 17.399s (17.399s including waiting). Image size: 1306417402 bytes.' metadata: creationTimestamp: "2026-04-24T19:16:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:47Z" name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c.18a96106326d4703 namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: 41fd19ab-1f32-4151-abb5-579750face73 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: f4821ffc-0241-4b44-9207-1e7f3b37cce3 kind: Event lastTimestamp: "2026-04-24T19:16:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:16:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:47Z" name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c.18a96106364ed2cc namespace: kserve-ci-e2e-test resourceVersion: "14384" uid: c3a942d7-6524-47a9-9fd1-c311e9fa3995 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: f4821ffc-0241-4b44-9207-1e7f3b37cce3 kind: Event lastTimestamp: "2026-04-24T19:16:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:16:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:47Z" name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c.18a9610636c86254 namespace: kserve-ci-e2e-test resourceVersion: "14385" uid: a99ee34e-ec4e-440f-ad68-0cc2b64654ca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: f4821ffc-0241-4b44-9207-1e7f3b37cce3 kind: Event lastTimestamp: "2026-04-24T19:16:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:16:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:47Z" name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c.18a9610636d586a4 namespace: kserve-ci-e2e-test resourceVersion: "14386" uid: a0cb86c8-ab32-4593-b027-7ccaf60ca78a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: f4821ffc-0241-4b44-9207-1e7f3b37cce3 kind: Event lastTimestamp: "2026-04-24T19:16:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:16:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:47Z" name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c.18a961063a58ecf7 namespace: kserve-ci-e2e-test resourceVersion: "14388" uid: 435830c0-634c-4fad-8ad0-bfd28c68d952 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: f4821ffc-0241-4b44-9207-1e7f3b37cce3 kind: Event lastTimestamp: "2026-04-24T19:16:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:16:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:16:47Z" name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c.18a961063ad120ef namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: 890785bb-f6ef-4d37-8db8-c2be8aea3eea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T19:16:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: f4821ffc-0241-4b44-9207-1e7f3b37cce3 kind: Event lastTimestamp: "2026-04-24T19:17:44Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:16:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:17:44Z" name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c.18a9610696f860f2 namespace: kserve-ci-e2e-test resourceVersion: "14817" uid: a9fb6758-8a81-47d9-9aea-2d593ecb5fad reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: f4821ffc-0241-4b44-9207-1e7f3b37cce3 kind: Event lastTimestamp: "2026-04-24T19:18:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:15Z" name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c.18a9611aa9ed09de namespace: kserve-ci-e2e-test resourceVersion: "15201" uid: 16ff6a47-9daf-4762-9aba-63a7b42ee908 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: f4821ffc-0241-4b44-9207-1e7f3b37cce3 kind: Event lastTimestamp: "2026-04-24T19:18:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:15Z" name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c.18a9611aa9ed8e1b namespace: kserve-ci-e2e-test resourceVersion: "15204" uid: ce49f3b0-873c-4463-8c31-aaef07e8f054 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77 namespace: kserve-ci-e2e-test resourceVersion: "14131" uid: 41ad9793-ef06-42a5-acf0-17be7b125c47 kind: Event lastTimestamp: "2026-04-24T19:16:24Z" message: 'Created pod: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77-k5p6c' metadata: creationTimestamp: "2026-04-24T19:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:16:24Z" name: isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77.18a96100fa708359 namespace: kserve-ci-e2e-test resourceVersion: "14148" uid: d0c543a1-045b-4ece-950b-661686190207 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-1c30c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14127" uid: f2ab02b6-fec0-4449-abd8-0668e8524a53 kind: Event lastTimestamp: "2026-04-24T19:16:24Z" message: Scaled up replica set isvc-xgboost-graph-raw-1c30c-predictor-6cb5798b77 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:16:24Z" name: isvc-xgboost-graph-raw-1c30c-predictor.18a96100f7bbedca namespace: kserve-ci-e2e-test resourceVersion: "14135" uid: 1e235717-547e-4166-9fcd-fd933bcc611f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T19:16:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-1c30c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14129" uid: 2efb7c23-7259-45fe-bd55-69adb63a9ac0 kind: Event lastTimestamp: "2026-04-24T19:17:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:16:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:17:24Z" name: isvc-xgboost-graph-raw-1c30c-predictor.18a96104766c5768 namespace: kserve-ci-e2e-test resourceVersion: "14672" uid: 3c896e62-4bd8-46fe-9da4-66e91a386ef9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T19:16:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-1c30c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14129" uid: 2efb7c23-7259-45fe-bd55-69adb63a9ac0 kind: Event lastTimestamp: "2026-04-24T19:17:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:16:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:17:24Z" name: isvc-xgboost-graph-raw-1c30c-predictor.18a9610476e0b84a namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: 675549b1-3852-473a-90e8-16b10bfd2667 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:17:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-1c30c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14329" uid: 2efb7c23-7259-45fe-bd55-69adb63a9ac0 kind: Event lastTimestamp: "2026-04-24T19:18:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:17:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:18:09Z" name: isvc-xgboost-graph-raw-1c30c-predictor.18a9611271d3d6ce namespace: kserve-ci-e2e-test resourceVersion: "15055" uid: 8320eab4-94ef-4a56-be07-36a93181695d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:17:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-1c30c-predictor namespace: kserve-ci-e2e-test resourceVersion: "14329" uid: 2efb7c23-7259-45fe-bd55-69adb63a9ac0 kind: Event lastTimestamp: "2026-04-24T19:18:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:17:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:18:09Z" name: isvc-xgboost-graph-raw-1c30c-predictor.18a9611272448b8d namespace: kserve-ci-e2e-test resourceVersion: "15056" uid: 8e8260a0-9470-465c-bf26-64b65e823411 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T19:16:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-1c30c namespace: kserve-ci-e2e-test resourceVersion: "14115" uid: 0c2d43dc-30c8-4d18-a938-39346eb42537 kind: Event lastTimestamp: "2026-04-24T19:17:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:17:55Z" name: isvc-xgboost-graph-raw-1c30c.18a96100f7b89f1f namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 97f9faca-767a-4f95-a1f7-4dc5a72d2716 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:16:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-1c30c namespace: kserve-ci-e2e-test resourceVersion: "14115" uid: 0c2d43dc-30c8-4d18-a938-39346eb42537 kind: Event lastTimestamp: "2026-04-24T19:16:24Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-1c30c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-1c30c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:16:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:16:24Z" name: isvc-xgboost-graph-raw-1c30c.18a96100f858a3bc namespace: kserve-ci-e2e-test resourceVersion: "14141" uid: 5485c474-a650-47ab-bf2c-c0ad7818efa6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:17:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-1c30c namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: 0c2d43dc-30c8-4d18-a938-39346eb42537 kind: Event lastTimestamp: "2026-04-24T19:17:54Z" message: InferenceService [isvc-xgboost-graph-raw-1c30c] is Ready metadata: creationTimestamp: "2026-04-24T19:17:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:17:54Z" name: isvc-xgboost-graph-raw-1c30c.18a96115f9d85304 namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: 2abd8ab0-178a-4851-a019-72aca2e15439 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:18:15.187890Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76 namespace: kserve-ci-e2e-test resourceVersion: "15207" uid: 73108d1e-c23f-4757-b27f-857d1b472a6c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76 to ip-10-0-138-52.ec2.internal metadata: creationTimestamp: "2026-04-24T19:18:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:18:15Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76.18a9611aadd603e2 namespace: kserve-ci-e2e-test resourceVersion: "15211" uid: f72f89fd-0422-4e21-be36-3c6100341548 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df6b44bbb-hm2bm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:15Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 73108d1e-c23f-4757-b27f-857d1b472a6c kind: Event lastTimestamp: "2026-04-24T19:18:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-hpa-1911c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:15Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76.18a9611ab7f32ba0 namespace: kserve-ci-e2e-test resourceVersion: "15222" uid: 3f9ffff4-7183-4695-99da-c6969f8c53a8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76 namespace: kserve-ci-e2e-test resourceVersion: "15215" uid: 73108d1e-c23f-4757-b27f-857d1b472a6c kind: Event lastTimestamp: "2026-04-24T19:18:16Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:18:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:18:16Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76.18a9611aeb5620b3 namespace: kserve-ci-e2e-test resourceVersion: "15238" uid: b8a708c5-b28f-49d2-9dd8-071b777be5d0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 73108d1e-c23f-4757-b27f-857d1b472a6c kind: Event lastTimestamp: "2026-04-24T19:18:16Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0183e14c0f4ec83676f700a2c092ab4e3599fa115c6cab4546b23e5207166882" already present on machine metadata: creationTimestamp: "2026-04-24T19:18:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:16Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76.18a9611aed5892c5 namespace: kserve-ci-e2e-test resourceVersion: "15240" uid: 79a0ea7e-50c3-4330-ab1a-cb6c8f7fc763 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 73108d1e-c23f-4757-b27f-857d1b472a6c kind: Event lastTimestamp: "2026-04-24T19:18:16Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T19:18:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:16Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76.18a9611af16d7d5f namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 74837699-e3fc-4816-8b72-2ed70fb460c0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 73108d1e-c23f-4757-b27f-857d1b472a6c kind: Event lastTimestamp: "2026-04-24T19:18:16Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T19:18:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:16Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76.18a9611af1e83527 namespace: kserve-ci-e2e-test resourceVersion: "15245" uid: 6359bd88-8078-4332-a307-048bb57398a8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 73108d1e-c23f-4757-b27f-857d1b472a6c kind: Event lastTimestamp: "2026-04-24T19:18:21Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-24T19:18:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:21Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76.18a9611c194c9237 namespace: kserve-ci-e2e-test resourceVersion: "15315" uid: bb250423-2c5d-4dfc-9db6-4f66fa50f419 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 73108d1e-c23f-4757-b27f-857d1b472a6c kind: Event lastTimestamp: "2026-04-24T19:18:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:18:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:21Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76.18a9611c1ccbd047 namespace: kserve-ci-e2e-test resourceVersion: "15320" uid: 17b74042-6932-4eef-8a71-5cfd026b9f04 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 73108d1e-c23f-4757-b27f-857d1b472a6c kind: Event lastTimestamp: "2026-04-24T19:18:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:18:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:21Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76.18a9611c1d57e86a namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 65c5d042-e769-41e7-80ed-8c0cd57215c2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 73108d1e-c23f-4757-b27f-857d1b472a6c kind: Event lastTimestamp: "2026-04-24T19:18:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:18:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:21Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76.18a9611c1d663b22 namespace: kserve-ci-e2e-test resourceVersion: "15324" uid: 6338dfd6-f23d-4e2e-a64a-8cfe05ab007a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 73108d1e-c23f-4757-b27f-857d1b472a6c kind: Event lastTimestamp: "2026-04-24T19:18:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:18:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:21Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76.18a9611c20fcd643 namespace: kserve-ci-e2e-test resourceVersion: "15327" uid: a5889489-843a-477d-ba1a-54ff67d40ee3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 73108d1e-c23f-4757-b27f-857d1b472a6c kind: Event lastTimestamp: "2026-04-24T19:18:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:18:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:21Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76.18a9611c218687cf namespace: kserve-ci-e2e-test resourceVersion: "15328" uid: 15358945-7f1a-4822-8b28-f297d4ccc588 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T19:18:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 73108d1e-c23f-4757-b27f-857d1b472a6c kind: Event lastTimestamp: "2026-04-24T19:19:58Z" message: 'Readiness probe failed: dial tcp 10.132.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:18:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:19:58Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76.18a9611c554737de namespace: kserve-ci-e2e-test resourceVersion: "16198" uid: db42c48e-0a25-41ad-8384-fa3ec19e11b5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 73108d1e-c23f-4757-b27f-857d1b472a6c kind: Event lastTimestamp: "2026-04-24T19:19:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:19:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:19:55Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76.18a9613202813e02 namespace: kserve-ci-e2e-test resourceVersion: "16154" uid: b8927833-abaf-47b0-8574-9029ce4e38f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 73108d1e-c23f-4757-b27f-857d1b472a6c kind: Event lastTimestamp: "2026-04-24T19:19:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:19:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:19:55Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76.18a961320281b4a3 namespace: kserve-ci-e2e-test resourceVersion: "16155" uid: cdcc62a4-def3-4f03-bed9-7ebc8fb9fc1b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 73108d1e-c23f-4757-b27f-857d1b472a6c kind: Event lastTimestamp: "2026-04-24T19:19:58Z" message: 'Readiness probe failed: Get "https://10.132.0.43:8643/healthz": dial tcp 10.132.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:19:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:19:58Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76.18a96132af8c461d namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: e331e788-a5d9-4cf3-bea6-89197d86b424 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc namespace: kserve-ci-e2e-test resourceVersion: "15169" uid: 8f7bcbbf-f858-47ff-be9f-d303a05934b8 kind: Event lastTimestamp: "2026-04-24T19:18:15Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc-gqc76' metadata: creationTimestamp: "2026-04-24T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:18:15Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc.18a9611aad6fe3e0 namespace: kserve-ci-e2e-test resourceVersion: "15208" uid: 4ed7728e-bd36-42fe-82da-57a1c06f2021 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-1911c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15165" uid: 4c3d4845-c7ec-4de9-976f-f657e9f2686e kind: Event lastTimestamp: "2026-04-24T19:18:15Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-1911c-predictor-86b5f986fc from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:18:15Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor.18a9611aa6e1786a namespace: kserve-ci-e2e-test resourceVersion: "15175" uid: 7e35f670-177a-4eaf-a11d-be00c557fd6c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-1911c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15168" uid: 1516a2f8-bc40-4e80-9de5-b7a6834e1bca kind: Event lastTimestamp: "2026-04-24T19:18:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:18:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:18:30Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor.18a9611e256049e3 namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: b35d79bd-ebb7-444d-88c0-da36ded731c8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-1911c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15168" uid: 1516a2f8-bc40-4e80-9de5-b7a6834e1bca kind: Event lastTimestamp: "2026-04-24T19:18:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:18:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:18:30Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor.18a9611e25cb519e namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: 2a160238-d6ab-4686-ac12-ef14da400df5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:18:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-1911c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15398" uid: 1516a2f8-bc40-4e80-9de5-b7a6834e1bca kind: Event lastTimestamp: "2026-04-24T19:19:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:18:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:19:45Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor.18a96121a48d9873 namespace: kserve-ci-e2e-test resourceVersion: "15980" uid: 7a1c5c98-4c1f-4489-984b-9b08dbd5d96b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T19:18:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-1911c-predictor namespace: kserve-ci-e2e-test resourceVersion: "15398" uid: 1516a2f8-bc40-4e80-9de5-b7a6834e1bca kind: Event lastTimestamp: "2026-04-24T19:19:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:18:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:19:45Z" name: isvc-xgboost-graph-raw-hpa-1911c-predictor.18a96121a4fce916 namespace: kserve-ci-e2e-test resourceVersion: "15981" uid: 40c3081f-e843-402e-8c86-a5478b0ef6f0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T19:18:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-1911c namespace: kserve-ci-e2e-test resourceVersion: "15156" uid: 0a1cb1f6-e852-4fb2-a55e-d6c6ec2f756f kind: Event lastTimestamp: "2026-04-24T19:19:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:19:28Z" name: isvc-xgboost-graph-raw-hpa-1911c.18a9611aa6b7e215 namespace: kserve-ci-e2e-test resourceVersion: "15833" uid: b4ea8e9b-c79f-47b0-a1ff-f217ad87a96e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-1911c namespace: kserve-ci-e2e-test resourceVersion: "15828" uid: 0a1cb1f6-e852-4fb2-a55e-d6c6ec2f756f kind: Event lastTimestamp: "2026-04-24T19:19:28Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-1911c] is Ready metadata: creationTimestamp: "2026-04-24T19:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:19:28Z" name: isvc-xgboost-graph-raw-hpa-1911c.18a9612bb8a8fab8 namespace: kserve-ci-e2e-test resourceVersion: "15829" uid: 812928a6-f448-48c0-aac4-e18b5833bd12 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:19:55.330080Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: c364dfc2-55dc-4e24-b4b8-27eee8cc3660 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv to ip-10-0-138-52.ec2.internal metadata: creationTimestamp: "2026-04-24T19:19:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:19:55Z" name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv.18a96131fec69345 namespace: kserve-ci-e2e-test resourceVersion: "16130" uid: d8c1b072-9959-48fa-a604-1533d30d639f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df6b44bbb-hm2bm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:55Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv namespace: kserve-ci-e2e-test resourceVersion: "16135" uid: c364dfc2-55dc-4e24-b4b8-27eee8cc3660 kind: Event lastTimestamp: "2026-04-24T19:19:55Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:19:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:19:55Z" name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv.18a96132187f7ee4 namespace: kserve-ci-e2e-test resourceVersion: "16162" uid: 9858f0d0-dcd5-441a-9d70-207f6a3f402e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: c364dfc2-55dc-4e24-b4b8-27eee8cc3660 kind: Event lastTimestamp: "2026-04-24T19:19:55Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-24T19:19:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:19:55Z" name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv.18a961321a020a42 namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: b1af8123-86fa-4119-b548-8decf2b31aad reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: c364dfc2-55dc-4e24-b4b8-27eee8cc3660 kind: Event lastTimestamp: "2026-04-24T19:19:56Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.071s (1.071s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-24T19:19:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:19:56Z" name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv.18a9613259e48983 namespace: kserve-ci-e2e-test resourceVersion: "16172" uid: 4f25d6ea-edef-4971-8127-59b3d9d2db2e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: c364dfc2-55dc-4e24-b4b8-27eee8cc3660 kind: Event lastTimestamp: "2026-04-24T19:19:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:19:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:19:56Z" name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv.18a961325d28e9ed namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: c9b64ae5-1ef4-447b-b6b3-e9bb949a2da7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: c364dfc2-55dc-4e24-b4b8-27eee8cc3660 kind: Event lastTimestamp: "2026-04-24T19:19:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:19:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:19:56Z" name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv.18a961325d9ee23a namespace: kserve-ci-e2e-test resourceVersion: "16175" uid: 40fe3012-1253-4e32-88f7-6c9164e12963 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: c364dfc2-55dc-4e24-b4b8-27eee8cc3660 kind: Event lastTimestamp: "2026-04-24T19:19:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:19:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:19:56Z" name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv.18a961325daba125 namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: f3930c48-9434-450a-82e2-bd5a30018148 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: c364dfc2-55dc-4e24-b4b8-27eee8cc3660 kind: Event lastTimestamp: "2026-04-24T19:19:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:19:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:19:56Z" name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv.18a96132619180ed namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: 478d2cba-1fd8-4283-9c15-f39f5e14feb0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: c364dfc2-55dc-4e24-b4b8-27eee8cc3660 kind: Event lastTimestamp: "2026-04-24T19:19:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:19:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:19:57Z" name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv.18a96132624ae815 namespace: kserve-ci-e2e-test resourceVersion: "16180" uid: a796bcbf-1646-4392-9486-280d7e7f25f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:21:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: c364dfc2-55dc-4e24-b4b8-27eee8cc3660 kind: Event lastTimestamp: "2026-04-24T19:21:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:21:40Z" name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv.18a9614a8694713c namespace: kserve-ci-e2e-test resourceVersion: "17099" uid: 857f0b9f-5169-42b3-8803-59b3d82503db reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:21:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: c364dfc2-55dc-4e24-b4b8-27eee8cc3660 kind: Event lastTimestamp: "2026-04-24T19:21:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:21:40Z" name: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv.18a9614a8695ab8a namespace: kserve-ci-e2e-test resourceVersion: "17100" uid: 503b4bb2-e799-4f9b-882b-cdc36bf1a267 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-2ea81-predictor-6b67b99d4c namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: 6a2e89bf-4cc7-4ef2-bae1-dde4f7cabf07 kind: Event lastTimestamp: "2026-04-24T19:19:55Z" message: 'Created pod: message-dumper-raw-2ea81-predictor-6b67b99d4c-rw8sv' metadata: creationTimestamp: "2026-04-24T19:19:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:19:55Z" name: message-dumper-raw-2ea81-predictor-6b67b99d4c.18a96131fe623768 namespace: kserve-ci-e2e-test resourceVersion: "16126" uid: a871743c-2a5d-4ed6-a16b-448583d57108 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-2ea81-predictor namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: f9e7434e-7f59-4fcb-853e-27d7596feb8d kind: Event lastTimestamp: "2026-04-24T19:19:55Z" message: Scaled up replica set message-dumper-raw-2ea81-predictor-6b67b99d4c from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:19:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:19:55Z" name: message-dumper-raw-2ea81-predictor.18a96131fb182c23 namespace: kserve-ci-e2e-test resourceVersion: "16106" uid: 42d9d317-f001-4e0a-a932-ab1b0b243940 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:20:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-2ea81-predictor namespace: kserve-ci-e2e-test resourceVersion: "16100" uid: 477e46d4-faac-426a-a882-78937aeaabb4 kind: Event lastTimestamp: "2026-04-24T19:20:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:20:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:20:40Z" name: message-dumper-raw-2ea81-predictor.18a9613579ca0d59 namespace: kserve-ci-e2e-test resourceVersion: "16592" uid: cb0d756f-a088-470d-8685-f9487036fa53 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:20:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-2ea81-predictor namespace: kserve-ci-e2e-test resourceVersion: "16100" uid: 477e46d4-faac-426a-a882-78937aeaabb4 kind: Event lastTimestamp: "2026-04-24T19:20:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:20:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:20:40Z" name: message-dumper-raw-2ea81-predictor.18a961357a3b5b4d namespace: kserve-ci-e2e-test resourceVersion: "16594" uid: 30aa124c-7eae-4c46-87a4-b67ea5905798 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:20:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-2ea81-predictor namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 477e46d4-faac-426a-a882-78937aeaabb4 kind: Event lastTimestamp: "2026-04-24T19:20:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:20:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:20:25Z" name: message-dumper-raw-2ea81-predictor.18a96138f95fce74 namespace: kserve-ci-e2e-test resourceVersion: "16477" uid: a1fb2e09-d790-4cd3-9e8a-aed0b290b702 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:20:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-2ea81-predictor namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 477e46d4-faac-426a-a882-78937aeaabb4 kind: Event lastTimestamp: "2026-04-24T19:20:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:20:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:20:25Z" name: message-dumper-raw-2ea81-predictor.18a96138f9c84f04 namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 39b1ec7d-bdc0-4801-98f9-b17ab3644a5f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T19:19:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-2ea81 namespace: kserve-ci-e2e-test resourceVersion: "16088" uid: 005f4a68-246c-4f6c-8985-94d431b9ddf7 kind: Event lastTimestamp: "2026-04-24T19:20:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:19:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:20:05Z" name: message-dumper-raw-2ea81.18a96131fae3c7f7 namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 45f40b25-c789-4901-8dba-180f7b6992d2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-2ea81 namespace: kserve-ci-e2e-test resourceVersion: "16105" uid: 005f4a68-246c-4f6c-8985-94d431b9ddf7 kind: Event lastTimestamp: "2026-04-24T19:19:55Z" message: 'Failed to update status for InferenceService "message-dumper-raw-2ea81": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-2ea81": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:19:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:19:55Z" name: message-dumper-raw-2ea81.18a96131fe34f2a4 namespace: kserve-ci-e2e-test resourceVersion: "16125" uid: 1dbc6b25-375e-4a84-b04d-6740be5d2e2b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:20:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-2ea81 namespace: kserve-ci-e2e-test resourceVersion: "16268" uid: 005f4a68-246c-4f6c-8985-94d431b9ddf7 kind: Event lastTimestamp: "2026-04-24T19:20:05Z" message: InferenceService [message-dumper-raw-2ea81] is Ready metadata: creationTimestamp: "2026-04-24T19:20:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:20:05Z" name: message-dumper-raw-2ea81.18a961346bc8a228 namespace: kserve-ci-e2e-test resourceVersion: "16269" uid: abfc0bc8-52ae-4896-8ffc-66f8f503390c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:18:04.800458Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-1c30c-5c79844df6-j74n7 namespace: kserve-ci-e2e-test resourceVersion: "14994" uid: 696cf363-91e9-4d72-9ce7-9f97ba73d285 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-1c30c-5c79844df6-j74n7 to ip-10-0-138-52.ec2.internal metadata: creationTimestamp: "2026-04-24T19:18:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:18:04Z" name: model-chainer-raw-1c30c-5c79844df6-j74n7.18a9611842b25245 namespace: kserve-ci-e2e-test resourceVersion: "14999" uid: 6fc046a6-84ad-4327-be1f-423f0b0f491b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df6b44bbb-hm2bm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:05Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-1c30c-5c79844df6-j74n7 namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: 696cf363-91e9-4d72-9ce7-9f97ba73d285 kind: Event lastTimestamp: "2026-04-24T19:18:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-1c30c-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T19:18:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:05Z" name: model-chainer-raw-1c30c-5c79844df6-j74n7.18a96118523a25be namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: bd2307c9-c271-4dbe-9f0e-088be885770a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:05Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-1c30c-5c79844df6-j74n7 namespace: kserve-ci-e2e-test resourceVersion: "15002" uid: 696cf363-91e9-4d72-9ce7-9f97ba73d285 kind: Event lastTimestamp: "2026-04-24T19:18:05Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:18:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:18:05Z" name: model-chainer-raw-1c30c-5c79844df6-j74n7.18a961187fd691a8 namespace: kserve-ci-e2e-test resourceVersion: "15019" uid: 18277426-e445-4921-aa12-653a668b78ff reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-1c30c} kind: Pod name: model-chainer-raw-1c30c-5c79844df6-j74n7 namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: 696cf363-91e9-4d72-9ce7-9f97ba73d285 kind: Event lastTimestamp: "2026-04-24T19:18:05Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:434eda131522047dad0943830e77efdaea314155166dd4fe016a4bb35d654538" metadata: creationTimestamp: "2026-04-24T19:18:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:05Z" name: model-chainer-raw-1c30c-5c79844df6-j74n7.18a96118811d6581 namespace: kserve-ci-e2e-test resourceVersion: "15021" uid: fa116473-62e0-4a2c-9255-54ae877f425d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-1c30c} kind: Pod name: model-chainer-raw-1c30c-5c79844df6-j74n7 namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: 696cf363-91e9-4d72-9ce7-9f97ba73d285 kind: Event lastTimestamp: "2026-04-24T19:18:07Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:434eda131522047dad0943830e77efdaea314155166dd4fe016a4bb35d654538" in 2.023s (2.023s including waiting). Image size: 216346179 bytes.' metadata: creationTimestamp: "2026-04-24T19:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:07Z" name: model-chainer-raw-1c30c-5c79844df6-j74n7.18a96118f9bcef74 namespace: kserve-ci-e2e-test resourceVersion: "15035" uid: 90541adc-18ce-4611-a16d-5152514acd22 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-1c30c} kind: Pod name: model-chainer-raw-1c30c-5c79844df6-j74n7 namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: 696cf363-91e9-4d72-9ce7-9f97ba73d285 kind: Event lastTimestamp: "2026-04-24T19:18:07Z" message: 'Created container: model-chainer-raw-1c30c' metadata: creationTimestamp: "2026-04-24T19:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:07Z" name: model-chainer-raw-1c30c-5c79844df6-j74n7.18a96118fce83620 namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: 967a5834-27eb-4685-8076-2ca089801c0f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-1c30c} kind: Pod name: model-chainer-raw-1c30c-5c79844df6-j74n7 namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: 696cf363-91e9-4d72-9ce7-9f97ba73d285 kind: Event lastTimestamp: "2026-04-24T19:18:07Z" message: Started container model-chainer-raw-1c30c metadata: creationTimestamp: "2026-04-24T19:18:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:07Z" name: model-chainer-raw-1c30c-5c79844df6-j74n7.18a96118fd611d53 namespace: kserve-ci-e2e-test resourceVersion: "15038" uid: 9dd86809-7a34-4afb-b155-24b3d1e30d90 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-1c30c} kind: Pod name: model-chainer-raw-1c30c-5c79844df6-j74n7 namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: 696cf363-91e9-4d72-9ce7-9f97ba73d285 kind: Event lastTimestamp: "2026-04-24T19:18:14Z" message: Stopping container model-chainer-raw-1c30c metadata: creationTimestamp: "2026-04-24T19:18:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:14Z" name: model-chainer-raw-1c30c-5c79844df6-j74n7.18a9611a9ace7fd6 namespace: kserve-ci-e2e-test resourceVersion: "15113" uid: 5a0d333a-6613-42a4-bbe2-c45431e87a9a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:18:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-1c30c} kind: Pod name: model-chainer-raw-1c30c-5c79844df6-j74n7 namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: 696cf363-91e9-4d72-9ce7-9f97ba73d285 kind: Event lastTimestamp: "2026-04-24T19:18:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T19:18:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:18:44Z" name: model-chainer-raw-1c30c-5c79844df6-j74n7.18a9611b9df21b6b namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: fd577646-1ff3-45b7-b2da-c125381b3ebd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-1c30c-5c79844df6 namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: ed44f6ca-4d82-41f0-8744-002b0862bc9a kind: Event lastTimestamp: "2026-04-24T19:18:04Z" message: 'Created pod: model-chainer-raw-1c30c-5c79844df6-j74n7' metadata: creationTimestamp: "2026-04-24T19:18:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:18:04Z" name: model-chainer-raw-1c30c-5c79844df6.18a96118424d1020 namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: 7e2b8f42-b3f6-45ed-9016-94e315827042 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-1c30c namespace: kserve-ci-e2e-test resourceVersion: "14989" uid: 9735b03e-c6a9-4b6a-b66a-66f9f029639a kind: Event lastTimestamp: "2026-04-24T19:18:04Z" message: Scaled up replica set model-chainer-raw-1c30c-5c79844df6 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:18:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:18:04Z" name: model-chainer-raw-1c30c.18a96118417013a3 namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: 8da691b5-769a-4e4e-99d3-ea3682039f32 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:18:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-1c30c namespace: kserve-ci-e2e-test resourceVersion: "15005" uid: 5cca54a8-c3be-4f09-bfb9-1e6ec486d578 kind: Event lastTimestamp: "2026-04-24T19:18:04Z" message: InferenceGraph [model-chainer-raw-1c30c] is Ready metadata: creationTimestamp: "2026-04-24T19:18:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:18:04Z" name: model-chainer-raw-1c30c.18a96118461c6390 namespace: kserve-ci-e2e-test resourceVersion: "15007" uid: 2807e4b3-105f-4322-824a-eef6b726547a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:19:45.104879Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-1911c-589fc8b564-dz99v namespace: kserve-ci-e2e-test resourceVersion: "15967" uid: 36c4bc0b-9659-434f-aa75-1340d759cef0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-1911c-589fc8b564-dz99v to ip-10-0-137-23.ec2.internal metadata: creationTimestamp: "2026-04-24T19:19:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:19:45Z" name: model-chainer-raw-hpa-1911c-589fc8b564-dz99v.18a9612f9d4e6248 namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: 28b8a825-5c76-40b8-838a-f70db06781ed reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df6b44bbb-hm2bm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:45Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-1911c-589fc8b564-dz99v namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 36c4bc0b-9659-434f-aa75-1340d759cef0 kind: Event lastTimestamp: "2026-04-24T19:19:45Z" message: Add eth0 [10.134.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:19:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:19:45Z" name: model-chainer-raw-hpa-1911c-589fc8b564-dz99v.18a9612fb6870662 namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: f08a34a8-476d-41a2-ac52-d487e5adf4c2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-1911c} kind: Pod name: model-chainer-raw-hpa-1911c-589fc8b564-dz99v namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: 36c4bc0b-9659-434f-aa75-1340d759cef0 kind: Event lastTimestamp: "2026-04-24T19:19:45Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:434eda131522047dad0943830e77efdaea314155166dd4fe016a4bb35d654538" metadata: creationTimestamp: "2026-04-24T19:19:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:19:45Z" name: model-chainer-raw-hpa-1911c-589fc8b564-dz99v.18a9612fb7cd66d6 namespace: kserve-ci-e2e-test resourceVersion: "15995" uid: 04ff0941-8af5-449e-ba7d-e41d8b0a5c15 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-23.ec2.internal source: component: kubelet host: ip-10-0-137-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-1911c} kind: Pod name: model-chainer-raw-hpa-1911c-589fc8b564-dz99v namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: 36c4bc0b-9659-434f-aa75-1340d759cef0 kind: Event lastTimestamp: "2026-04-24T19:19:47Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:434eda131522047dad0943830e77efdaea314155166dd4fe016a4bb35d654538" in 2.345s (2.345s including waiting). Image size: 216346179 bytes.' metadata: creationTimestamp: "2026-04-24T19:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:19:47Z" name: model-chainer-raw-hpa-1911c-589fc8b564-dz99v.18a96130439eca1d namespace: kserve-ci-e2e-test resourceVersion: "16010" uid: c3c5870e-4e44-45b3-976b-a024896127cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-23.ec2.internal source: component: kubelet host: ip-10-0-137-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-1911c} kind: Pod name: model-chainer-raw-hpa-1911c-589fc8b564-dz99v namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: 36c4bc0b-9659-434f-aa75-1340d759cef0 kind: Event lastTimestamp: "2026-04-24T19:19:47Z" message: 'Created container: model-chainer-raw-hpa-1911c' metadata: creationTimestamp: "2026-04-24T19:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:19:47Z" name: model-chainer-raw-hpa-1911c-589fc8b564-dz99v.18a9613047b76bfe namespace: kserve-ci-e2e-test resourceVersion: "16011" uid: cf838a55-b931-481c-adbd-3dc3aea2a97a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-23.ec2.internal source: component: kubelet host: ip-10-0-137-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-1911c} kind: Pod name: model-chainer-raw-hpa-1911c-589fc8b564-dz99v namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: 36c4bc0b-9659-434f-aa75-1340d759cef0 kind: Event lastTimestamp: "2026-04-24T19:19:47Z" message: Started container model-chainer-raw-hpa-1911c metadata: creationTimestamp: "2026-04-24T19:19:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:19:47Z" name: model-chainer-raw-hpa-1911c-589fc8b564-dz99v.18a9613048338259 namespace: kserve-ci-e2e-test resourceVersion: "16012" uid: 949e8a29-a34f-466f-ae83-75197d0bc3eb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-23.ec2.internal source: component: kubelet host: ip-10-0-137-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-1911c} kind: Pod name: model-chainer-raw-hpa-1911c-589fc8b564-dz99v namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: 36c4bc0b-9659-434f-aa75-1340d759cef0 kind: Event lastTimestamp: "2026-04-24T19:19:55Z" message: Stopping container model-chainer-raw-hpa-1911c metadata: creationTimestamp: "2026-04-24T19:19:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:19:55Z" name: model-chainer-raw-hpa-1911c-589fc8b564-dz99v.18a96131f53f3c22 namespace: kserve-ci-e2e-test resourceVersion: "16084" uid: ac8bba62-a04b-482e-b608-271c4a864558 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-23.ec2.internal source: component: kubelet host: ip-10-0-137-23.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T19:19:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-1911c} kind: Pod name: model-chainer-raw-hpa-1911c-589fc8b564-dz99v namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: 36c4bc0b-9659-434f-aa75-1340d759cef0 kind: Event lastTimestamp: "2026-04-24T19:20:24Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T19:19:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:20:24Z" name: model-chainer-raw-hpa-1911c-589fc8b564-dz99v.18a96132eebcc2ab namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 33d9d2c8-1cf1-43de-b68a-7e1b3789ca01 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-23.ec2.internal source: component: kubelet host: ip-10-0-137-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-1911c-589fc8b564 namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: 801ffc7c-00af-4443-a189-26231674f46b kind: Event lastTimestamp: "2026-04-24T19:19:45Z" message: 'Created pod: model-chainer-raw-hpa-1911c-589fc8b564-dz99v' metadata: creationTimestamp: "2026-04-24T19:19:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:19:45Z" name: model-chainer-raw-hpa-1911c-589fc8b564.18a9612f9ce6b5c4 namespace: kserve-ci-e2e-test resourceVersion: "15969" uid: d462d05d-90ec-45b0-b3e6-2d6d5e6806e3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-1911c namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 1ad17d53-d1a3-4774-a634-a3a8d5fa70c6 kind: Event lastTimestamp: "2026-04-24T19:19:45Z" message: Scaled up replica set model-chainer-raw-hpa-1911c-589fc8b564 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:19:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:19:45Z" name: model-chainer-raw-hpa-1911c.18a9612f9be60355 namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: a409c22e-e3f5-42c3-a38b-08cef9b7fac0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:19:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-1911c namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: 384d82d6-09b6-433f-8430-a12bb2e990a6 kind: Event lastTimestamp: "2026-04-24T19:19:45Z" message: InferenceGraph [model-chainer-raw-hpa-1911c] is Ready metadata: creationTimestamp: "2026-04-24T19:19:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:19:45Z" name: model-chainer-raw-hpa-1911c.18a9612fa0a3ee4e namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: b8643baf-6af1-4456-a1fd-cd6a54fc05d3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:25:52.343892Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6 namespace: kserve-ci-e2e-test resourceVersion: "19239" uid: 144f8023-7c54-4bff-bb03-2c1bf181a4c6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-eaa35-predictor-5f65c47b68-v94x6 to ip-10-0-138-52.ec2.internal metadata: creationTimestamp: "2026-04-24T19:25:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:25:52Z" name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6.18a961851e74f2b3 namespace: kserve-ci-e2e-test resourceVersion: "19244" uid: d79fb878-9df0-4b3e-a614-23c1aba7aeeb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df6b44bbb-hm2bm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:52Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6 namespace: kserve-ci-e2e-test resourceVersion: "19241" uid: 144f8023-7c54-4bff-bb03-2c1bf181a4c6 kind: Event lastTimestamp: "2026-04-24T19:25:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-eaa35-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T19:25:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:52Z" name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6.18a96185285e7318 namespace: kserve-ci-e2e-test resourceVersion: "19252" uid: 64f17d38-c957-4fb5-80ee-a95f22915cd7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:53Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6 namespace: kserve-ci-e2e-test resourceVersion: "19249" uid: 144f8023-7c54-4bff-bb03-2c1bf181a4c6 kind: Event lastTimestamp: "2026-04-24T19:25:53Z" message: Add eth0 [10.132.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:25:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:25:53Z" name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6.18a961855c015882 namespace: kserve-ci-e2e-test resourceVersion: "19266" uid: e3e298b3-9743-4671-aaba-de503d646db4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6 namespace: kserve-ci-e2e-test resourceVersion: "19241" uid: 144f8023-7c54-4bff-bb03-2c1bf181a4c6 kind: Event lastTimestamp: "2026-04-24T19:25:53Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0183e14c0f4ec83676f700a2c092ab4e3599fa115c6cab4546b23e5207166882" already present on machine metadata: creationTimestamp: "2026-04-24T19:25:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:53Z" name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6.18a961855d6d85ab namespace: kserve-ci-e2e-test resourceVersion: "19268" uid: 8b58b2a9-0779-4639-8798-e601e6eab40e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6 namespace: kserve-ci-e2e-test resourceVersion: "19241" uid: 144f8023-7c54-4bff-bb03-2c1bf181a4c6 kind: Event lastTimestamp: "2026-04-24T19:25:53Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T19:25:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:53Z" name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6.18a96185610d2f70 namespace: kserve-ci-e2e-test resourceVersion: "19269" uid: 13a268ec-e7cf-4ce3-8d29-3894ee63bdd7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:53Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6 namespace: kserve-ci-e2e-test resourceVersion: "19241" uid: 144f8023-7c54-4bff-bb03-2c1bf181a4c6 kind: Event lastTimestamp: "2026-04-24T19:25:53Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T19:25:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:53Z" name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6.18a961856186e709 namespace: kserve-ci-e2e-test resourceVersion: "19270" uid: 56deec95-b1bd-421c-aba5-ed17af21f026 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6 namespace: kserve-ci-e2e-test resourceVersion: "19241" uid: 144f8023-7c54-4bff-bb03-2c1bf181a4c6 kind: Event lastTimestamp: "2026-04-24T19:25:58Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:25:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:58Z" name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6.18a9618671a989e1 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 70532c44-ab5e-40a3-85b0-9eb7df652364 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6 namespace: kserve-ci-e2e-test resourceVersion: "19241" uid: 144f8023-7c54-4bff-bb03-2c1bf181a4c6 kind: Event lastTimestamp: "2026-04-24T19:25:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:25:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:58Z" name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6.18a9618675a6f619 namespace: kserve-ci-e2e-test resourceVersion: "19313" uid: 88f85fd4-7f96-4b8a-b1b4-cc1fb785cfca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6 namespace: kserve-ci-e2e-test resourceVersion: "19241" uid: 144f8023-7c54-4bff-bb03-2c1bf181a4c6 kind: Event lastTimestamp: "2026-04-24T19:25:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:25:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:58Z" name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6.18a96186762723b9 namespace: kserve-ci-e2e-test resourceVersion: "19315" uid: 083d5e70-2e37-4827-8995-f47b6d25d070 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6 namespace: kserve-ci-e2e-test resourceVersion: "19241" uid: 144f8023-7c54-4bff-bb03-2c1bf181a4c6 kind: Event lastTimestamp: "2026-04-24T19:25:58Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:25:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:58Z" name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6.18a961867634dba3 namespace: kserve-ci-e2e-test resourceVersion: "19316" uid: 880b2fb7-0ae1-4a7e-95f1-1c7f0e90052c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6 namespace: kserve-ci-e2e-test resourceVersion: "19241" uid: 144f8023-7c54-4bff-bb03-2c1bf181a4c6 kind: Event lastTimestamp: "2026-04-24T19:25:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:25:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:58Z" name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6.18a961867a205065 namespace: kserve-ci-e2e-test resourceVersion: "19319" uid: c5ca5bb5-2acd-4127-af0a-2b1bb7cd8bb3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6 namespace: kserve-ci-e2e-test resourceVersion: "19241" uid: 144f8023-7c54-4bff-bb03-2c1bf181a4c6 kind: Event lastTimestamp: "2026-04-24T19:25:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:25:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:25:58Z" name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6.18a961867a9bcb4b namespace: kserve-ci-e2e-test resourceVersion: "19320" uid: ae35431a-90a5-4e3d-b86b-1f15a460ea43 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T19:26:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6 namespace: kserve-ci-e2e-test resourceVersion: "19241" uid: 144f8023-7c54-4bff-bb03-2c1bf181a4c6 kind: Event lastTimestamp: "2026-04-24T19:27:16Z" message: 'Readiness probe failed: dial tcp 10.132.0.50:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:27:16Z" name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6.18a96186e9892d24 namespace: kserve-ci-e2e-test resourceVersion: "19953" uid: b4c027c6-aa74-4ee2-931e-a552ad120188 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6 namespace: kserve-ci-e2e-test resourceVersion: "19241" uid: 144f8023-7c54-4bff-bb03-2c1bf181a4c6 kind: Event lastTimestamp: "2026-04-24T19:27:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:27:12Z" name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6.18a96197c66047e3 namespace: kserve-ci-e2e-test resourceVersion: "19878" uid: 516a5c0c-3874-451e-98ef-0041d917f354 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6 namespace: kserve-ci-e2e-test resourceVersion: "19241" uid: 144f8023-7c54-4bff-bb03-2c1bf181a4c6 kind: Event lastTimestamp: "2026-04-24T19:27:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:27:12Z" name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6.18a96197c660c01f namespace: kserve-ci-e2e-test resourceVersion: "19881" uid: 96e5a8bd-dd03-419e-a824-11d467a44835 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6 namespace: kserve-ci-e2e-test resourceVersion: "19241" uid: 144f8023-7c54-4bff-bb03-2c1bf181a4c6 kind: Event lastTimestamp: "2026-04-24T19:27:16Z" message: 'Readiness probe failed: Get "https://10.132.0.50:8643/healthz": dial tcp 10.132.0.50:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:27:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:27:16Z" name: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6.18a961989bb1edb9 namespace: kserve-ci-e2e-test resourceVersion: "19952" uid: 1af920d2-c4b7-4411-a250-5e2db6b3e86e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-eaa35-predictor-5f65c47b68 namespace: kserve-ci-e2e-test resourceVersion: "19230" uid: f1c84fc2-5ed4-4193-975e-8efdea91941f kind: Event lastTimestamp: "2026-04-24T19:25:52Z" message: 'Created pod: raw-sklearn-eaa35-predictor-5f65c47b68-v94x6' metadata: creationTimestamp: "2026-04-24T19:25:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:52Z" name: raw-sklearn-eaa35-predictor-5f65c47b68.18a961851dd6b413 namespace: kserve-ci-e2e-test resourceVersion: "19240" uid: f0fc25ce-7a38-4684-b02b-67b6fe709d3a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-eaa35-predictor namespace: kserve-ci-e2e-test resourceVersion: "19228" uid: 601b14be-b0a5-4b48-bb6d-7ce0e3d3b89c kind: Event lastTimestamp: "2026-04-24T19:25:52Z" message: Scaled up replica set raw-sklearn-eaa35-predictor-5f65c47b68 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:25:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:25:52Z" name: raw-sklearn-eaa35-predictor.18a961851b7a5593 namespace: kserve-ci-e2e-test resourceVersion: "19232" uid: 93f9559a-523d-46e4-9df2-3f0d82e9a83c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:26:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-eaa35-predictor namespace: kserve-ci-e2e-test resourceVersion: "19229" uid: 2f773966-9f55-4a7d-a201-751f9c0d714c kind: Event lastTimestamp: "2026-04-24T19:26:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:26:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:37Z" name: raw-sklearn-eaa35-predictor.18a961889a06d034 namespace: kserve-ci-e2e-test resourceVersion: "19607" uid: c9fb3581-2f5d-4dbf-814d-9b1ef6931317 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:26:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-eaa35-predictor namespace: kserve-ci-e2e-test resourceVersion: "19229" uid: 2f773966-9f55-4a7d-a201-751f9c0d714c kind: Event lastTimestamp: "2026-04-24T19:26:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:26:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:26:37Z" name: raw-sklearn-eaa35-predictor.18a961889a84204e namespace: kserve-ci-e2e-test resourceVersion: "19608" uid: 3fe4c1bc-6435-41e6-b361-3c3e11f6aebb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:26:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-eaa35-predictor namespace: kserve-ci-e2e-test resourceVersion: "19395" uid: 2f773966-9f55-4a7d-a201-751f9c0d714c kind: Event lastTimestamp: "2026-04-24T19:27:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:27:07Z" name: raw-sklearn-eaa35-predictor.18a9619317d48e8c namespace: kserve-ci-e2e-test resourceVersion: "19828" uid: d36703c0-2e16-467a-b5d1-b8c0bf65b4b8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:26:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-eaa35-predictor namespace: kserve-ci-e2e-test resourceVersion: "19395" uid: 2f773966-9f55-4a7d-a201-751f9c0d714c kind: Event lastTimestamp: "2026-04-24T19:27:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:26:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:27:07Z" name: raw-sklearn-eaa35-predictor.18a96193184329d2 namespace: kserve-ci-e2e-test resourceVersion: "19829" uid: 6984a1ae-f066-4a10-8662-b0c7c8a3d100 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-24T19:25:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-eaa35 namespace: kserve-ci-e2e-test resourceVersion: "19218" uid: 664025f7-a6b7-4515-bbe7-ba04da3625d3 kind: Event lastTimestamp: "2026-04-24T19:27:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:25:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:27:06Z" name: raw-sklearn-eaa35.18a961851b5b7f3f namespace: kserve-ci-e2e-test resourceVersion: "19816" uid: 79748e8c-8afd-4a8f-80cf-608a45cee9bd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:25:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-eaa35 namespace: kserve-ci-e2e-test resourceVersion: "19218" uid: 664025f7-a6b7-4515-bbe7-ba04da3625d3 kind: Event lastTimestamp: "2026-04-24T19:25:52Z" message: 'Failed to update status for InferenceService "raw-sklearn-eaa35": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-eaa35": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:25:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:25:52Z" name: raw-sklearn-eaa35.18a961851bd883a5 namespace: kserve-ci-e2e-test resourceVersion: "19234" uid: 4450ca89-23df-4513-8614-d9ede728c323 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-eaa35 namespace: kserve-ci-e2e-test resourceVersion: "19813" uid: 664025f7-a6b7-4515-bbe7-ba04da3625d3 kind: Event lastTimestamp: "2026-04-24T19:27:06Z" message: InferenceService [raw-sklearn-eaa35] is Ready metadata: creationTimestamp: "2026-04-24T19:27:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:27:06Z" name: raw-sklearn-eaa35.18a961964d0b8de2 namespace: kserve-ci-e2e-test resourceVersion: "19815" uid: 48d2e213-d592-4077-aa5c-16ec595116f2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T19:27:12.567271Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8 namespace: kserve-ci-e2e-test resourceVersion: "19902" uid: 4bb62ecb-cff8-41d5-aafb-38182389d863 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8 to ip-10-0-138-52.ec2.internal metadata: creationTimestamp: "2026-04-24T19:27:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T19:27:12Z" name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8.18a96197cc24a64f namespace: kserve-ci-e2e-test resourceVersion: "19906" uid: 7660a9ad-c84c-473f-b7fd-5c6afea957ec reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5df6b44bbb-hm2bm source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:13Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8 namespace: kserve-ci-e2e-test resourceVersion: "19912" uid: 4bb62ecb-cff8-41d5-aafb-38182389d863 kind: Event lastTimestamp: "2026-04-24T19:27:13Z" message: Add eth0 [10.132.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T19:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T19:27:13Z" name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8.18a96197e759af03 namespace: kserve-ci-e2e-test resourceVersion: "19921" uid: 687ec52e-7149-464b-9800-26d6212613a7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8 namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: 4bb62ecb-cff8-41d5-aafb-38182389d863 kind: Event lastTimestamp: "2026-04-24T19:27:13Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0183e14c0f4ec83676f700a2c092ab4e3599fa115c6cab4546b23e5207166882" already present on machine metadata: creationTimestamp: "2026-04-24T19:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:27:13Z" name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8.18a96197e8fc4357 namespace: kserve-ci-e2e-test resourceVersion: "19923" uid: 008f8dde-a0af-4887-af5e-89d396631358 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8 namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: 4bb62ecb-cff8-41d5-aafb-38182389d863 kind: Event lastTimestamp: "2026-04-24T19:27:13Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T19:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:27:13Z" name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8.18a96197ec572936 namespace: kserve-ci-e2e-test resourceVersion: "19925" uid: f1c843eb-5e36-49b5-ab26-83aaaeeed62c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8 namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: 4bb62ecb-cff8-41d5-aafb-38182389d863 kind: Event lastTimestamp: "2026-04-24T19:27:13Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T19:27:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:27:13Z" name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8.18a96197ecd95e6d namespace: kserve-ci-e2e-test resourceVersion: "19926" uid: 10a5e630-14ee-4e4f-99af-c6cb86ff2b3c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8 namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: 4bb62ecb-cff8-41d5-aafb-38182389d863 kind: Event lastTimestamp: "2026-04-24T19:27:17Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1445" already present on machine metadata: creationTimestamp: "2026-04-24T19:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:27:17Z" name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8.18a96198e8f4dc30 namespace: kserve-ci-e2e-test resourceVersion: "19968" uid: 0be26f9a-84e5-4efa-a3ea-5e6e91ebb59d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8 namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: 4bb62ecb-cff8-41d5-aafb-38182389d863 kind: Event lastTimestamp: "2026-04-24T19:27:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T19:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:27:17Z" name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8.18a96198ecbb2a52 namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: d3a14c07-e54e-4e7d-8440-a75d1ae48168 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8 namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: 4bb62ecb-cff8-41d5-aafb-38182389d863 kind: Event lastTimestamp: "2026-04-24T19:27:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T19:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:27:17Z" name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8.18a96198ed4ae36e namespace: kserve-ci-e2e-test resourceVersion: "19975" uid: 2bc80c09-b254-45c0-bb49-aecc615747fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8 namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: 4bb62ecb-cff8-41d5-aafb-38182389d863 kind: Event lastTimestamp: "2026-04-24T19:27:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T19:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:27:17Z" name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8.18a96198ed59afa1 namespace: kserve-ci-e2e-test resourceVersion: "19976" uid: 8b02f443-7ced-470d-a7a1-89dc6b4496b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8 namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: 4bb62ecb-cff8-41d5-aafb-38182389d863 kind: Event lastTimestamp: "2026-04-24T19:27:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T19:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:27:17Z" name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8.18a96198f149655b namespace: kserve-ci-e2e-test resourceVersion: "19979" uid: d7b35070-34d6-4166-9301-47caf85af0fd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8 namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: 4bb62ecb-cff8-41d5-aafb-38182389d863 kind: Event lastTimestamp: "2026-04-24T19:27:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:27:17Z" name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8.18a96198f1c2d74c namespace: kserve-ci-e2e-test resourceVersion: "19980" uid: 62826be3-f23a-4d91-a4ce-656fe1159b65 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T19:27:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8 namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: 4bb62ecb-cff8-41d5-aafb-38182389d863 kind: Event lastTimestamp: "2026-04-24T19:28:35Z" message: 'Readiness probe failed: dial tcp 10.132.0.51:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:28:35Z" name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8.18a9619960ca945a namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 70f9b043-06e5-463a-84fc-d28c619944b7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:28:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8 namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: 4bb62ecb-cff8-41d5-aafb-38182389d863 kind: Event lastTimestamp: "2026-04-24T19:28:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T19:28:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:28:32Z" name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8.18a961aa7173d9a1 namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: b37e8415-a0d0-4120-9183-39d8b108d1ba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:28:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8 namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: 4bb62ecb-cff8-41d5-aafb-38182389d863 kind: Event lastTimestamp: "2026-04-24T19:28:32Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T19:28:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:28:32Z" name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8.18a961aa71749087 namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: 067624ee-3190-4371-b8d2-8eda1fac4ac7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:28:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8 namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: 4bb62ecb-cff8-41d5-aafb-38182389d863 kind: Event lastTimestamp: "2026-04-24T19:28:35Z" message: 'Readiness probe failed: Get "https://10.132.0.51:8643/healthz": dial tcp 10.132.0.51:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T19:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T19:28:35Z" name: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8.18a961ab12e2a5bd namespace: kserve-ci-e2e-test resourceVersion: "20593" uid: e0c1b564-4867-44ab-8867-1e5c1add180c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-52.ec2.internal source: component: kubelet host: ip-10-0-138-52.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-63372-predictor-5c59d6df78 namespace: kserve-ci-e2e-test resourceVersion: "19891" uid: 50e62dc7-13f7-4f62-ad4d-af689979fe15 kind: Event lastTimestamp: "2026-04-24T19:27:12Z" message: 'Created pod: raw-sklearn-runtime-63372-predictor-5c59d6df78-zvnw8' metadata: creationTimestamp: "2026-04-24T19:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:27:12Z" name: raw-sklearn-runtime-63372-predictor-5c59d6df78.18a96197cbbb99e6 namespace: kserve-ci-e2e-test resourceVersion: "19903" uid: dcf364f4-cc38-4afd-9502-c9d1a6ea7cfa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-63372-predictor namespace: kserve-ci-e2e-test resourceVersion: "19889" uid: 35a5a8f6-4eb1-4dad-ae38-ed26de8b0637 kind: Event lastTimestamp: "2026-04-24T19:27:12Z" message: Scaled up replica set raw-sklearn-runtime-63372-predictor-5c59d6df78 from 0 to 1 metadata: creationTimestamp: "2026-04-24T19:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:27:12Z" name: raw-sklearn-runtime-63372-predictor.18a96197c91a2c56 namespace: kserve-ci-e2e-test resourceVersion: "19894" uid: c54e3095-2156-4453-8da9-b14a2f3e41c6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:27:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-63372-predictor namespace: kserve-ci-e2e-test resourceVersion: "19890" uid: 2fc055c4-db2d-49d2-9ac2-8d845a7e98ca kind: Event lastTimestamp: "2026-04-24T19:27:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:27:42Z" name: raw-sklearn-runtime-63372-predictor.18a9619b4815372c namespace: kserve-ci-e2e-test resourceVersion: "20157" uid: 92708ebb-d26a-471a-b3c3-5971af93dc42 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T19:27:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-63372-predictor namespace: kserve-ci-e2e-test resourceVersion: "19890" uid: 2fc055c4-db2d-49d2-9ac2-8d845a7e98ca kind: Event lastTimestamp: "2026-04-24T19:27:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T19:27:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:27:42Z" name: raw-sklearn-runtime-63372-predictor.18a9619b4880ee07 namespace: kserve-ci-e2e-test resourceVersion: "20158" uid: ec005e1e-683e-453d-98a0-6b4ede0a2874 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:27:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-63372-predictor namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 2fc055c4-db2d-49d2-9ac2-8d845a7e98ca kind: Event lastTimestamp: "2026-04-24T19:28:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:27:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:28:27Z" name: raw-sklearn-runtime-63372-predictor.18a961a246bef205 namespace: kserve-ci-e2e-test resourceVersion: "20488" uid: 2a2b9ac3-9440-4752-9383-0e5565825d8c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T19:27:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-63372-predictor namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 2fc055c4-db2d-49d2-9ac2-8d845a7e98ca kind: Event lastTimestamp: "2026-04-24T19:28:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T19:27:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T19:28:27Z" name: raw-sklearn-runtime-63372-predictor.18a961a2472594df namespace: kserve-ci-e2e-test resourceVersion: "20489" uid: b570afe9-3c2f-400a-ad04-3043da931f3e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-24T19:27:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-63372 namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: 98a42697-e63a-4510-ba26-f1e23784bb3b kind: Event lastTimestamp: "2026-04-24T19:28:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T19:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:28:25Z" name: raw-sklearn-runtime-63372.18a96197c8fc7628 namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: f60a2476-556a-401a-8028-8aec35f78c4f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:27:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-63372 namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: 98a42697-e63a-4510-ba26-f1e23784bb3b kind: Event lastTimestamp: "2026-04-24T19:27:12Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-63372": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-63372": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T19:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:27:12Z" name: raw-sklearn-runtime-63372.18a96197c94bf2b3 namespace: kserve-ci-e2e-test resourceVersion: "19896" uid: f5065d9f-66c4-4a80-b874-7084b0b46829 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T19:28:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-63372 namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: 98a42697-e63a-4510-ba26-f1e23784bb3b kind: Event lastTimestamp: "2026-04-24T19:28:25Z" message: InferenceService [raw-sklearn-runtime-63372] is Ready metadata: creationTimestamp: "2026-04-24T19:28:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T19:28:25Z" name: raw-sklearn-runtime-63372.18a961a8c3c3cd80 namespace: kserve-ci-e2e-test resourceVersion: "20474" uid: 6298f59d-0746-473c-b5c9-34ea76f9c29e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "20741"