--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:51:26.176318Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-223bfc-predictor-86594bf96-xczs2 namespace: kserve-ci-e2e-test resourceVersion: "19308" uid: 08252d44-397b-4cac-bf76-e4f024214bbc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-223bfc-predictor-86594bf96-xczs2 to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-04-23T13:51:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:51:26Z" name: isvc-init-fail-223bfc-predictor-86594bf96-xczs2.18a900b0885bc21e namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: ea16c2ee-dcb9-447b-82c9-3ac1f73d9de8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-78b4f5d84d-ssxfd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:26Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-223bfc-predictor-86594bf96-xczs2 namespace: kserve-ci-e2e-test resourceVersion: "19312" uid: 08252d44-397b-4cac-bf76-e4f024214bbc kind: Event lastTimestamp: "2026-04-23T13:51:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-init-fail-223bfc-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T13:51:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:26Z" name: isvc-init-fail-223bfc-predictor-86594bf96-xczs2.18a900b093a60517 namespace: kserve-ci-e2e-test resourceVersion: "19323" uid: 54ca5c9d-91de-4304-8b15-8dc02b7192fe reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:27Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-223bfc-predictor-86594bf96-xczs2 namespace: kserve-ci-e2e-test resourceVersion: "19318" uid: 08252d44-397b-4cac-bf76-e4f024214bbc kind: Event lastTimestamp: "2026-04-23T13:51:27Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:51:27Z" name: isvc-init-fail-223bfc-predictor-86594bf96-xczs2.18a900b0c70064e0 namespace: kserve-ci-e2e-test resourceVersion: "19335" uid: d8713432-db6f-4a24-923b-9b7ebf951029 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:51:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-223bfc-predictor-86594bf96-xczs2 namespace: kserve-ci-e2e-test resourceVersion: "19312" uid: 08252d44-397b-4cac-bf76-e4f024214bbc kind: Event lastTimestamp: "2026-04-23T13:51:34Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bfc8f044ee2a84e90b7b9876040a412b44d7b121a59a5fe7b22a946b15b61519" already present on machine metadata: creationTimestamp: "2026-04-23T13:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:34Z" name: isvc-init-fail-223bfc-predictor-86594bf96-xczs2.18a900b0c86317c5 namespace: kserve-ci-e2e-test resourceVersion: "19398" uid: 2f1a6ea4-4299-4d19-9e8f-801806b54fec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:51:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-223bfc-predictor-86594bf96-xczs2 namespace: kserve-ci-e2e-test resourceVersion: "19312" uid: 08252d44-397b-4cac-bf76-e4f024214bbc kind: Event lastTimestamp: "2026-04-23T13:51:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T13:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:34Z" name: isvc-init-fail-223bfc-predictor-86594bf96-xczs2.18a900b0cc618b66 namespace: kserve-ci-e2e-test resourceVersion: "19402" uid: 6b25a4cd-2f6f-4f9c-b170-085ca84afc17 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:51:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-223bfc-predictor-86594bf96-xczs2 namespace: kserve-ci-e2e-test resourceVersion: "19312" uid: 08252d44-397b-4cac-bf76-e4f024214bbc kind: Event lastTimestamp: "2026-04-23T13:51:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T13:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:34Z" name: isvc-init-fail-223bfc-predictor-86594bf96-xczs2.18a900b0cce811e8 namespace: kserve-ci-e2e-test resourceVersion: "19403" uid: 62935786-7224-4a28-99ab-e064fd4963ac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-223bfc-predictor-86594bf96-xczs2 namespace: kserve-ci-e2e-test resourceVersion: "19312" uid: 08252d44-397b-4cac-bf76-e4f024214bbc kind: Event lastTimestamp: "2026-04-23T13:51:36Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-04-23T13:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:36Z" name: isvc-init-fail-223bfc-predictor-86594bf96-xczs2.18a900b2ef44fc7f namespace: kserve-ci-e2e-test resourceVersion: "19442" uid: e244bd4c-d70f-4ef4-9de3-19635a8367ec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-223bfc-predictor-86594bf96 namespace: kserve-ci-e2e-test resourceVersion: "19301" uid: 9405d5ba-f8a1-4dbf-9f09-2a0ab7f9ba6c kind: Event lastTimestamp: "2026-04-23T13:51:26Z" message: 'Created pod: isvc-init-fail-223bfc-predictor-86594bf96-xczs2' metadata: creationTimestamp: "2026-04-23T13:51:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:51:26Z" name: isvc-init-fail-223bfc-predictor-86594bf96.18a900b087f4779f namespace: kserve-ci-e2e-test resourceVersion: "19311" uid: ab7e9883-56f3-4777-b34b-9f12ec2de368 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-223bfc-predictor namespace: kserve-ci-e2e-test resourceVersion: "19298" uid: 9a73945d-700b-4c66-98e6-e1751d7e3973 kind: Event lastTimestamp: "2026-04-23T13:51:26Z" message: Scaled up replica set isvc-init-fail-223bfc-predictor-86594bf96 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:51:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:51:26Z" name: isvc-init-fail-223bfc-predictor.18a900b0851a55fc namespace: kserve-ci-e2e-test resourceVersion: "19303" uid: 2cdfbe33-c850-4ea6-8bea-cd2db817adb7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T13:51:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-223bfc namespace: kserve-ci-e2e-test resourceVersion: "19274" uid: 4877d9f7-2318-4da9-9393-e8c73a565b1d kind: Event lastTimestamp: "2026-04-23T13:51:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:51:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:51:34Z" name: isvc-init-fail-223bfc.18a900b084fa615e namespace: kserve-ci-e2e-test resourceVersion: "19404" uid: bdd8dad7-bbae-49a5-ac4e-30a97cd4f6e5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-223bfc namespace: kserve-ci-e2e-test resourceVersion: "19274" uid: 4877d9f7-2318-4da9-9393-e8c73a565b1d kind: Event lastTimestamp: "2026-04-23T13:51:26Z" message: 'Failed to update status for InferenceService "isvc-init-fail-223bfc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-223bfc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:51:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:51:26Z" name: isvc-init-fail-223bfc.18a900b085a54e93 namespace: kserve-ci-e2e-test resourceVersion: "19305" uid: a9f3dd53-33e6-4ae4-a573-4dcf1de54ceb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-223bfc namespace: kserve-ci-e2e-test resourceVersion: "19274" uid: 4877d9f7-2318-4da9-9393-e8c73a565b1d kind: Event lastTimestamp: "2026-04-23T13:51:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-223bfc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:51:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:51:26Z" name: isvc-init-fail-223bfc.18a900b085a5af36 namespace: kserve-ci-e2e-test resourceVersion: "19307" uid: 9cf5afa7-5db9-4eb4-8d80-93c7e919c548 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:45:44.241057Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16478" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-04-23T13:45:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:45:44Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a90060eb6db278 namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 0f8cffda-aec4-4ecf-9340-407af139f726 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-78b4f5d84d-ssxfd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:44Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: "2026-04-23T13:45:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-raw-35dde-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T13:45:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:44Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a90060f618cfeb namespace: kserve-ci-e2e-test resourceVersion: "16493" uid: 9546839f-3f7b-49d6-a64b-b6aadd13c287 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16487" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: "2026-04-23T13:45:45Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:45:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:45:45Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a9006128d836c2 namespace: kserve-ci-e2e-test resourceVersion: "16503" uid: 2c6cd48c-2456-47d1-a956-83543b502f73 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: "2026-04-23T13:45:45Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bfc8f044ee2a84e90b7b9876040a412b44d7b121a59a5fe7b22a946b15b61519" already present on machine metadata: creationTimestamp: "2026-04-23T13:45:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:45Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a900612a58e2dd namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 1b23b5de-cba7-4e98-8459-397d130b5846 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: "2026-04-23T13:45:45Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T13:45:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:45Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a900612d0efe56 namespace: kserve-ci-e2e-test resourceVersion: "16507" uid: 7bcc577f-c20b-48eb-aac1-67bbae268813 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: "2026-04-23T13:45:45Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T13:45:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:45Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a900612d86c1c0 namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: a690ef92-743a-4bc9-bb15-0f67caf823b5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: "2026-04-23T13:45:49Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T13:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:49Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a9006213febfe6 namespace: kserve-ci-e2e-test resourceVersion: "16545" uid: ca3022fb-1888-4a96-848a-eef7f99ed0a7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: "2026-04-23T13:45:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:49Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a90062177b68de namespace: kserve-ci-e2e-test resourceVersion: "16553" uid: cbd8cf2c-2d15-45f8-8183-da00a22cfa36 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: "2026-04-23T13:45:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:49Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a900621802cd2b namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: 27b7e218-e9c7-47b9-a721-e1567030aed1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: "2026-04-23T13:45:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:49Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a90062180f9e49 namespace: kserve-ci-e2e-test resourceVersion: "16555" uid: 6e03dee0-0ddf-4b53-972c-889dc2ae9fc2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: "2026-04-23T13:45:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:49Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a900621b8dc3e7 namespace: kserve-ci-e2e-test resourceVersion: "16557" uid: edfb509e-3075-466f-a62d-0d6f89418b37 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: "2026-04-23T13:45:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:49Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a900621c06fc20 namespace: kserve-ci-e2e-test resourceVersion: "16558" uid: b0bf2d5f-e282-4a52-80f7-c30db9689a49 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: "2026-04-23T13:45:49Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:d5d470ad0a2dbd76829def7f459909c05cc870ffb330e4b585204605f3c490d9" already present on machine metadata: creationTimestamp: "2026-04-23T13:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:49Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a900621c130a4c namespace: kserve-ci-e2e-test resourceVersion: "16559" uid: 3a7c8200-ec72-4f9b-8c3c-a3b812607920 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: "2026-04-23T13:45:49Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-23T13:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:49Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a900621f5370cd namespace: kserve-ci-e2e-test resourceVersion: "16560" uid: 62168a75-bc39-412b-848a-d35d6c127011 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: "2026-04-23T13:45:49Z" message: Started container agent metadata: creationTimestamp: "2026-04-23T13:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:49Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a900621fce2046 namespace: kserve-ci-e2e-test resourceVersion: "16561" uid: 2606432c-aa10-4a64-af3f-463291be7d06 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T13:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: "2026-04-23T13:47:26Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:47:26Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a900624ff7159b namespace: kserve-ci-e2e-test resourceVersion: "17382" uid: f7f5b11b-7ccf-4ebb-8dd0-b6c940f8f2d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T13:45:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: "2026-04-23T13:47:26Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T13:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:47:26Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a90062500a2c86 namespace: kserve-ci-e2e-test resourceVersion: "17383" uid: 16bba516-0f54-4b89-9068-d83a7ac20093 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:47:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: "2026-04-23T13:47:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:47:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:47:09Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a90074c4458ba0 namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: f2953a74-012f-4fff-a7af-796a6fdc31ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:47:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: "2026-04-23T13:47:09Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-23T13:47:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:47:09Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a90074c445ba95 namespace: kserve-ci-e2e-test resourceVersion: "17185" uid: 1020a288-463b-411c-b38d-4d5c095644df reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:47:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: "2026-04-23T13:47:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:47:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:47:09Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a90074c446bacc namespace: kserve-ci-e2e-test resourceVersion: "17188" uid: 740be4c0-dbe7-40bd-8efa-2e44499c9104 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:47:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: e9d0c878-c391-4fa0-9049-42aecf64aa28 kind: Event lastTimestamp: "2026-04-23T13:47:26Z" message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp 10.134.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:47:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:47:26Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl.18a900752c138f0e namespace: kserve-ci-e2e-test resourceVersion: "17381" uid: f0909320-b0c7-4d40-800b-f8dd3a55924a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: 89b5826c-0e41-4a05-8d19-a29bcbddf279 kind: Event lastTimestamp: "2026-04-23T13:45:44Z" message: 'Created pod: isvc-logger-raw-35dde-predictor-6dbbcf9c6d-xhlcl' metadata: creationTimestamp: "2026-04-23T13:45:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:44Z" name: isvc-logger-raw-35dde-predictor-6dbbcf9c6d.18a90060eb079a76 namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 6e0bc898-c6e3-4915-958e-b3c24990d6c4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-35dde-predictor namespace: kserve-ci-e2e-test resourceVersion: "16465" uid: d982cea5-9a64-429a-8ea0-6c90a2f39589 kind: Event lastTimestamp: "2026-04-23T13:45:44Z" message: Scaled up replica set isvc-logger-raw-35dde-predictor-6dbbcf9c6d from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:45:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:44Z" name: isvc-logger-raw-35dde-predictor.18a90060e804dec4 namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: bc483c35-0a91-4c9c-9d84-f61bc7b70618 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-35dde-predictor namespace: kserve-ci-e2e-test resourceVersion: "16466" uid: ca7565cd-5db3-46f2-ad7d-a99370b56c5d kind: Event lastTimestamp: "2026-04-23T13:45:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:45:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:59Z" name: isvc-logger-raw-35dde-predictor.18a9006466dcb1c7 namespace: kserve-ci-e2e-test resourceVersion: "16635" uid: 7af0f258-78c2-4216-ba41-0b35be6a7ffe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-35dde-predictor namespace: kserve-ci-e2e-test resourceVersion: "16466" uid: ca7565cd-5db3-46f2-ad7d-a99370b56c5d kind: Event lastTimestamp: "2026-04-23T13:45:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:45:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:59Z" name: isvc-logger-raw-35dde-predictor.18a9006467406f3b namespace: kserve-ci-e2e-test resourceVersion: "16637" uid: 2ffbbede-7aae-4fc3-aae6-6e3895c3f3c4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:46:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-35dde-predictor namespace: kserve-ci-e2e-test resourceVersion: "16636" uid: ca7565cd-5db3-46f2-ad7d-a99370b56c5d kind: Event lastTimestamp: "2026-04-23T13:46:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:46:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:46:59Z" name: isvc-logger-raw-35dde-predictor.18a90067e5f16608 namespace: kserve-ci-e2e-test resourceVersion: "17074" uid: 6cf7bf91-ecf6-4329-ad74-fd04a17ea4bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:46:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-35dde-predictor namespace: kserve-ci-e2e-test resourceVersion: "16636" uid: ca7565cd-5db3-46f2-ad7d-a99370b56c5d kind: Event lastTimestamp: "2026-04-23T13:46:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:46:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:46:59Z" name: isvc-logger-raw-35dde-predictor.18a90067e656eb84 namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: a1ef3389-1c64-4330-86ca-759dd4b73d82 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-23T13:45:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-35dde namespace: kserve-ci-e2e-test resourceVersion: "16455" uid: f94af3cf-969d-49ee-a39e-aa8385992f5a kind: Event lastTimestamp: "2026-04-23T13:46:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:45:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:46:56Z" name: isvc-logger-raw-35dde.18a90060e7e1fbc2 namespace: kserve-ci-e2e-test resourceVersion: "17054" uid: 0848363c-2cd0-4ab1-9f04-4255d499bbeb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-35dde namespace: kserve-ci-e2e-test resourceVersion: "16455" uid: f94af3cf-969d-49ee-a39e-aa8385992f5a kind: Event lastTimestamp: "2026-04-23T13:45:44Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-35dde": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-35dde": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:45:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:45:44Z" name: isvc-logger-raw-35dde.18a90060e86f954f namespace: kserve-ci-e2e-test resourceVersion: "16472" uid: bf76fb9b-235e-4d8a-a408-414a523fc7e5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:46:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-35dde namespace: kserve-ci-e2e-test resourceVersion: "17052" uid: f94af3cf-969d-49ee-a39e-aa8385992f5a kind: Event lastTimestamp: "2026-04-23T13:46:56Z" message: InferenceService [isvc-logger-raw-35dde] is Ready metadata: creationTimestamp: "2026-04-23T13:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:46:56Z" name: isvc-logger-raw-35dde.18a90071b426f7b2 namespace: kserve-ci-e2e-test resourceVersion: "17053" uid: f844b694-46b3-4302-a1bf-f3ca15aa6d79 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:49:49.828918Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw namespace: kserve-ci-e2e-test resourceVersion: "18444" uid: da669fe3-20f7-405f-8b6a-c75eeba0766a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-04-23T13:49:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:49:49Z" name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw.18a9009a199afd40 namespace: kserve-ci-e2e-test resourceVersion: "18448" uid: 96100b38-99d2-45fa-bb5f-10c533376658 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-78b4f5d84d-ssxfd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:49:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw namespace: kserve-ci-e2e-test resourceVersion: "18446" uid: da669fe3-20f7-405f-8b6a-c75eeba0766a kind: Event lastTimestamp: "2026-04-23T13:49:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-primary-2a679f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:49:50Z" name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw.18a9009a272e6b5e namespace: kserve-ci-e2e-test resourceVersion: "18458" uid: c62c72d7-0d6b-4655-acb6-dc76ffdf76f4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:49:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw namespace: kserve-ci-e2e-test resourceVersion: "18453" uid: da669fe3-20f7-405f-8b6a-c75eeba0766a kind: Event lastTimestamp: "2026-04-23T13:49:50Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:49:50Z" name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw.18a9009a56b826d6 namespace: kserve-ci-e2e-test resourceVersion: "18470" uid: 57bea21d-2278-42a2-a45b-ba861d895a89 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw namespace: kserve-ci-e2e-test resourceVersion: "18446" uid: da669fe3-20f7-405f-8b6a-c75eeba0766a kind: Event lastTimestamp: "2026-04-23T13:49:50Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bfc8f044ee2a84e90b7b9876040a412b44d7b121a59a5fe7b22a946b15b61519" already present on machine metadata: creationTimestamp: "2026-04-23T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:49:50Z" name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw.18a9009a5807f6f6 namespace: kserve-ci-e2e-test resourceVersion: "18472" uid: 68cc8884-a321-4f4e-ae91-4a039ad21c94 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw namespace: kserve-ci-e2e-test resourceVersion: "18446" uid: da669fe3-20f7-405f-8b6a-c75eeba0766a kind: Event lastTimestamp: "2026-04-23T13:49:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:49:50Z" name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw.18a9009a5b852251 namespace: kserve-ci-e2e-test resourceVersion: "18473" uid: 7b1dd871-1ef4-4b6e-9c0f-d015ecdba1e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:49:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw namespace: kserve-ci-e2e-test resourceVersion: "18446" uid: da669fe3-20f7-405f-8b6a-c75eeba0766a kind: Event lastTimestamp: "2026-04-23T13:49:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T13:49:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:49:50Z" name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw.18a9009a5bfac7e5 namespace: kserve-ci-e2e-test resourceVersion: "18474" uid: 93b035e1-ef79-4737-88b8-68b89fd5c976 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:49:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw namespace: kserve-ci-e2e-test resourceVersion: "18446" uid: da669fe3-20f7-405f-8b6a-c75eeba0766a kind: Event lastTimestamp: "2026-04-23T13:49:55Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T13:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:49:55Z" name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw.18a9009b54566cdd namespace: kserve-ci-e2e-test resourceVersion: "18511" uid: 14c5da6f-4d9e-49e5-9422-f1bf4c0e071c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:49:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw namespace: kserve-ci-e2e-test resourceVersion: "18446" uid: da669fe3-20f7-405f-8b6a-c75eeba0766a kind: Event lastTimestamp: "2026-04-23T13:49:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:49:55Z" name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw.18a9009b5884280d namespace: kserve-ci-e2e-test resourceVersion: "18516" uid: 1271ecdb-4a11-47d8-a242-b03e97ff192f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:49:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw namespace: kserve-ci-e2e-test resourceVersion: "18446" uid: da669fe3-20f7-405f-8b6a-c75eeba0766a kind: Event lastTimestamp: "2026-04-23T13:49:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:49:55Z" name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw.18a9009b5909d910 namespace: kserve-ci-e2e-test resourceVersion: "18517" uid: ecf5bc60-ceb6-40da-a3a2-9a153affed6c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:49:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw namespace: kserve-ci-e2e-test resourceVersion: "18446" uid: da669fe3-20f7-405f-8b6a-c75eeba0766a kind: Event lastTimestamp: "2026-04-23T13:49:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:49:55Z" name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw.18a9009b59165779 namespace: kserve-ci-e2e-test resourceVersion: "18518" uid: 5aa58063-62ce-42f2-ac15-34ae2f7b0d08 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:49:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw namespace: kserve-ci-e2e-test resourceVersion: "18446" uid: da669fe3-20f7-405f-8b6a-c75eeba0766a kind: Event lastTimestamp: "2026-04-23T13:49:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:49:55Z" name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw.18a9009b5c8b41dc namespace: kserve-ci-e2e-test resourceVersion: "18520" uid: a1fb4690-3894-4433-90ab-5afa0a98913e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:49:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw namespace: kserve-ci-e2e-test resourceVersion: "18446" uid: da669fe3-20f7-405f-8b6a-c75eeba0766a kind: Event lastTimestamp: "2026-04-23T13:49:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:49:55Z" name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw.18a9009b5d1046e6 namespace: kserve-ci-e2e-test resourceVersion: "18521" uid: 399d0f98-03b2-4889-8a68-b07e7a798b60 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-23T13:49:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw namespace: kserve-ci-e2e-test resourceVersion: "18446" uid: da669fe3-20f7-405f-8b6a-c75eeba0766a kind: Event lastTimestamp: "2026-04-23T13:50:52Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:50:52Z" name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw.18a9009b9056063b namespace: kserve-ci-e2e-test resourceVersion: "18930" uid: 81a35128-a708-4707-8179-f58a5f1b01f8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw namespace: kserve-ci-e2e-test resourceVersion: "18446" uid: da669fe3-20f7-405f-8b6a-c75eeba0766a kind: Event lastTimestamp: "2026-04-23T13:51:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:51:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:26Z" name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw.18a900b07ff6d79a namespace: kserve-ci-e2e-test resourceVersion: "19271" uid: 5c02eca2-ad67-4f52-ab1a-de888f2ee85a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw namespace: kserve-ci-e2e-test resourceVersion: "18446" uid: da669fe3-20f7-405f-8b6a-c75eeba0766a kind: Event lastTimestamp: "2026-04-23T13:51:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:51:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:26Z" name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw.18a900b07ff7738e namespace: kserve-ci-e2e-test resourceVersion: "19273" uid: 62d24e0a-b714-4369-a60d-24ee7db27bc6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw namespace: kserve-ci-e2e-test resourceVersion: "18446" uid: da669fe3-20f7-405f-8b6a-c75eeba0766a kind: Event lastTimestamp: "2026-04-23T13:51:27Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:27Z" name: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw.18a900b0c07c3bdf namespace: kserve-ci-e2e-test resourceVersion: "19334" uid: 211af6f5-0979-4533-8619-4c74c1bf02ca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:49:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-2a679f-predictor-f5ddf7b9c namespace: kserve-ci-e2e-test resourceVersion: "18435" uid: c66dcd61-4969-46f4-80c2-a745bfd04fa9 kind: Event lastTimestamp: "2026-04-23T13:49:49Z" message: 'Created pod: isvc-primary-2a679f-predictor-f5ddf7b9c-l2jxw' metadata: creationTimestamp: "2026-04-23T13:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:49:49Z" name: isvc-primary-2a679f-predictor-f5ddf7b9c.18a9009a19278e10 namespace: kserve-ci-e2e-test resourceVersion: "18445" uid: 54cf74d9-56e8-429b-9385-35c505d8f2bf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:49:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-2a679f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18432" uid: 7013be51-06d3-4899-a177-f3551d051632 kind: Event lastTimestamp: "2026-04-23T13:49:49Z" message: Scaled up replica set isvc-primary-2a679f-predictor-f5ddf7b9c from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:49:49Z" name: isvc-primary-2a679f-predictor.18a9009a1486eb8b namespace: kserve-ci-e2e-test resourceVersion: "18437" uid: a046c428-02e4-4a3c-ad45-a7847a1edf92 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:50:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-2a679f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18434" uid: 35b04c00-cdb5-40fe-9088-8888b3e5c009 kind: Event lastTimestamp: "2026-04-23T13:50:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:50:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:50:19Z" name: isvc-primary-2a679f-predictor.18a9009d93de5c7b namespace: kserve-ci-e2e-test resourceVersion: "18703" uid: 55d4ef53-a92e-44ed-b316-fcdcc4582073 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:50:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-2a679f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18434" uid: 35b04c00-cdb5-40fe-9088-8888b3e5c009 kind: Event lastTimestamp: "2026-04-23T13:50:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:50:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:50:19Z" name: isvc-primary-2a679f-predictor.18a9009d944e615f namespace: kserve-ci-e2e-test resourceVersion: "18704" uid: 1cb641a1-bc9a-4bb6-a6eb-943c177264b2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:50:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-2a679f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18595" uid: 35b04c00-cdb5-40fe-9088-8888b3e5c009 kind: Event lastTimestamp: "2026-04-23T13:51:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:50:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:51:19Z" name: isvc-primary-2a679f-predictor.18a900a492781567 namespace: kserve-ci-e2e-test resourceVersion: "19190" uid: 084204cb-36e0-4f71-bf26-6b7b41c55678 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:50:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-2a679f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18595" uid: 35b04c00-cdb5-40fe-9088-8888b3e5c009 kind: Event lastTimestamp: "2026-04-23T13:51:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:50:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:51:19Z" name: isvc-primary-2a679f-predictor.18a900a492e8aa88 namespace: kserve-ci-e2e-test resourceVersion: "19191" uid: 331cd523-f3eb-487c-8956-a4e784b94c15 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-23T13:49:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-2a679f namespace: kserve-ci-e2e-test resourceVersion: "18405" uid: c917d8fd-1a4e-47ae-933d-e5643f21d54b kind: Event lastTimestamp: "2026-04-23T13:51:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:51:02Z" name: isvc-primary-2a679f.18a9009a14540c0a namespace: kserve-ci-e2e-test resourceVersion: "19011" uid: a789923c-8be5-40c9-921c-de2362396f8e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:49:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-2a679f namespace: kserve-ci-e2e-test resourceVersion: "18405" uid: c917d8fd-1a4e-47ae-933d-e5643f21d54b kind: Event lastTimestamp: "2026-04-23T13:49:49Z" message: 'Failed to update status for InferenceService "isvc-primary-2a679f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-2a679f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:49:49Z" name: isvc-primary-2a679f.18a9009a14bf7395 namespace: kserve-ci-e2e-test resourceVersion: "18452" uid: fb17b724-e0a2-4acd-a4fa-08bdbe8771b0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:49:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-2a679f namespace: kserve-ci-e2e-test resourceVersion: "18405" uid: c917d8fd-1a4e-47ae-933d-e5643f21d54b kind: Event lastTimestamp: "2026-04-23T13:49:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-2a679f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:49:49Z" name: isvc-primary-2a679f.18a9009a14bfc6bb namespace: kserve-ci-e2e-test resourceVersion: "18455" uid: 555d79d6-e50f-4ddf-8d21-58571d2e60d8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-2a679f namespace: kserve-ci-e2e-test resourceVersion: "19008" uid: c917d8fd-1a4e-47ae-933d-e5643f21d54b kind: Event lastTimestamp: "2026-04-23T13:51:02Z" message: InferenceService [isvc-primary-2a679f] is Ready metadata: creationTimestamp: "2026-04-23T13:51:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:51:02Z" name: isvc-primary-2a679f.18a900aaf3777516 namespace: kserve-ci-e2e-test resourceVersion: "19009" uid: d58e91cb-26ea-427e-aa13-2877e1d6911f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:40:18.418002Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13459" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-04-23T13:40:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:40:18Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a900150edc5d07 namespace: kserve-ci-e2e-test resourceVersion: "13465" uid: 092a170d-c883-4bac-9d1e-96203e86db46 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-78b4f5d84d-ssxfd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:18Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13471" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:40:18Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:40:18Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a9001528adb8ed namespace: kserve-ci-e2e-test resourceVersion: "13480" uid: 1e29eb45-98cc-44a2-ad9e-9b94421d474e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:40:18Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:bfc8f044ee2a84e90b7b9876040a412b44d7b121a59a5fe7b22a946b15b61519" metadata: creationTimestamp: "2026-04-23T13:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:40:18Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a900152a273d46 namespace: kserve-ci-e2e-test resourceVersion: "13482" uid: 91419ac2-fc39-46b9-a613-7122c0a14d03 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:40:22Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:bfc8f044ee2a84e90b7b9876040a412b44d7b121a59a5fe7b22a946b15b61519" in 3.174s (3.174s including waiting). Image size: 301492121 bytes.' metadata: creationTimestamp: "2026-04-23T13:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:40:22Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a90015e7609b69 namespace: kserve-ci-e2e-test resourceVersion: "13503" uid: 4d1d0479-f000-4e43-9d4d-5127cf986b8b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:40:22Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T13:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:40:22Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a90015eaf5a93f namespace: kserve-ci-e2e-test resourceVersion: "13504" uid: b2301222-6d7d-406d-8f94-a02c734f1e5f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:40:22Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T13:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:40:22Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a90015eb6092c4 namespace: kserve-ci-e2e-test resourceVersion: "13505" uid: 00dce039-2245-4fd9-80c7-19949a518673 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:40:26Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1436" metadata: creationTimestamp: "2026-04-23T13:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:40:26Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a90016d4efc802 namespace: kserve-ci-e2e-test resourceVersion: "13541" uid: f04f592e-cbc5-46b9-a069-dd535038ebdd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:40:39Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1436" in 13.872s (13.872s including waiting). Image size: 1562812843 bytes.' metadata: creationTimestamp: "2026-04-23T13:40:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:40:39Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a9001a0fcbeb7b namespace: kserve-ci-e2e-test resourceVersion: "13647" uid: 09214f45-387c-4d2c-a53a-bd0b4e827c7d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:40:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:40:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:40:39Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a9001a130e51fd namespace: kserve-ci-e2e-test resourceVersion: "13648" uid: dfa9e4a0-0a6e-4589-addd-04e8677bdc76 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:40:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:40:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:40:39Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a9001a137bb5d8 namespace: kserve-ci-e2e-test resourceVersion: "13649" uid: 8226afac-ac59-4cc4-b943-10a9a4b0d33c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:40:39Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-23T13:40:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:40:39Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a9001a138815fc namespace: kserve-ci-e2e-test resourceVersion: "13650" uid: 88268ed2-95bf-4cc8-820d-2574fa5c1588 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:40:42Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.142s (2.142s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-23T13:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:40:42Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a9001a933ee6bf namespace: kserve-ci-e2e-test resourceVersion: "13663" uid: c401b700-18ac-4181-b6be-00179175087b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:40:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:40:42Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a9001a967b677d namespace: kserve-ci-e2e-test resourceVersion: "13664" uid: faa49ac1-e8fb-4f62-951f-d53edf15f911 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:40:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:40:42Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a9001a96f90255 namespace: kserve-ci-e2e-test resourceVersion: "13665" uid: 62b71488-5cc0-4e73-8d13-8a64e136e778 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:40:42Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:d5d470ad0a2dbd76829def7f459909c05cc870ffb330e4b585204605f3c490d9" metadata: creationTimestamp: "2026-04-23T13:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:40:42Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a9001a9708d3c7 namespace: kserve-ci-e2e-test resourceVersion: "13666" uid: 7643a5fc-6261-49f1-b1e0-16ca24bf3bcc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:40:44Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:d5d470ad0a2dbd76829def7f459909c05cc870ffb330e4b585204605f3c490d9" in 2.469s (2.469s including waiting). Image size: 238035008 bytes.' metadata: creationTimestamp: "2026-04-23T13:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:40:44Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a9001b2a396be4 namespace: kserve-ci-e2e-test resourceVersion: "13683" uid: 729e6fdd-db6b-4fcd-b0d4-18828a3a9fca reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:40:44Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-23T13:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:40:44Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a9001b2d594a57 namespace: kserve-ci-e2e-test resourceVersion: "13685" uid: 5da1ae69-cfd2-40a6-8abb-11d38270551c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:40:44Z" message: Started container agent metadata: creationTimestamp: "2026-04-23T13:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:40:44Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a9001b2deae030 namespace: kserve-ci-e2e-test resourceVersion: "13686" uid: 377ecaea-ccbe-47d0-b6a5-a887904fc655 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-23T13:40:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:42:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:18Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a9001b4658a4ac namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: 42b88521-86d5-4039-ba58-6d29fd381f58 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T13:40:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:42:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T13:40:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:18Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a9001b8222f5cd namespace: kserve-ci-e2e-test resourceVersion: "14507" uid: ccac802e-7f4a-43f5-82c1-dd528dac1177 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:42:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:03Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a9002d81f417bd namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 005bae42-07b1-47ec-9152-aba09b513a89 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:42:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:03Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a9002d81fd2dfb namespace: kserve-ci-e2e-test resourceVersion: "14284" uid: 1803bd5c-9b71-4a3a-87c1-62c19a998891 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:42:03Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-23T13:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:03Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a9002d81fff812 namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: da6c4bae-b3fb-42f7-b35b-9391290dd233 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:42:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: c5c6f975-4af9-4a6c-b5c6-abe6fc720e00 kind: Event lastTimestamp: "2026-04-23T13:42:22Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:42:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:22Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8.18a9002e5e455f69 namespace: kserve-ci-e2e-test resourceVersion: "14545" uid: cdfc4c22-78b4-43a5-8224-82b8528701c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68 namespace: kserve-ci-e2e-test resourceVersion: "13448" uid: def1f31e-acdc-4e4f-98e8-4851372dbb77 kind: Event lastTimestamp: "2026-04-23T13:40:18Z" message: 'Created pod: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68-29tt8' metadata: creationTimestamp: "2026-04-23T13:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:40:18Z" name: isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68.18a900150dd92271 namespace: kserve-ci-e2e-test resourceVersion: "13461" uid: 78824a85-3cc7-4d20-9bc3-0c0e308de31a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-ee9be-predictor namespace: kserve-ci-e2e-test resourceVersion: "13447" uid: bce44ce9-20f1-41f3-b1ed-2316e88b0904 kind: Event lastTimestamp: "2026-04-23T13:40:18Z" message: Scaled up replica set isvc-raw-sklearn-batcher-ee9be-predictor-7868f7cf68 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:40:18Z" name: isvc-raw-sklearn-batcher-ee9be-predictor.18a9001506065496 namespace: kserve-ci-e2e-test resourceVersion: "13449" uid: 4f484fba-146d-4018-a6e0-2421306ce420 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:40:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-ee9be-predictor namespace: kserve-ci-e2e-test resourceVersion: "13456" uid: 56694b13-4477-470a-b4c4-58df33e8efb0 kind: Event lastTimestamp: "2026-04-23T13:41:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:41:03Z" name: isvc-raw-sklearn-batcher-ee9be-predictor.18a900188b9865e9 namespace: kserve-ci-e2e-test resourceVersion: "13829" uid: 9fd86a45-359f-4370-a1f6-37804ae81205 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:40:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-ee9be-predictor namespace: kserve-ci-e2e-test resourceVersion: "13456" uid: 56694b13-4477-470a-b4c4-58df33e8efb0 kind: Event lastTimestamp: "2026-04-23T13:41:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:41:03Z" name: isvc-raw-sklearn-batcher-ee9be-predictor.18a900188c057946 namespace: kserve-ci-e2e-test resourceVersion: "13830" uid: 8838edfe-9fcf-4021-b77e-a7accb723c2b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:41:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-ee9be-predictor namespace: kserve-ci-e2e-test resourceVersion: "13600" uid: 56694b13-4477-470a-b4c4-58df33e8efb0 kind: Event lastTimestamp: "2026-04-23T13:41:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:41:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:41:48Z" name: isvc-raw-sklearn-batcher-ee9be-predictor.18a900230af41014 namespace: kserve-ci-e2e-test resourceVersion: "14152" uid: 8cdc9aa6-1eac-428a-953c-84952dd18c73 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:41:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-ee9be-predictor namespace: kserve-ci-e2e-test resourceVersion: "13600" uid: 56694b13-4477-470a-b4c4-58df33e8efb0 kind: Event lastTimestamp: "2026-04-23T13:41:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:41:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:41:48Z" name: isvc-raw-sklearn-batcher-ee9be-predictor.18a900230b7d22f8 namespace: kserve-ci-e2e-test resourceVersion: "14153" uid: f2ad5fd6-7465-458b-be08-d7ab46ece698 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-23T13:40:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-ee9be namespace: kserve-ci-e2e-test resourceVersion: "13435" uid: a0e25726-303a-4888-9eff-ebb378c4e8dd kind: Event lastTimestamp: "2026-04-23T13:41:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:41:48Z" name: isvc-raw-sklearn-batcher-ee9be.18a900150c41e013 namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: fbd46a4a-9741-4b94-8b22-eef6be945dca reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-ee9be namespace: kserve-ci-e2e-test resourceVersion: "13435" uid: a0e25726-303a-4888-9eff-ebb378c4e8dd kind: Event lastTimestamp: "2026-04-23T13:40:18Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-ee9be": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-ee9be": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:40:18Z" name: isvc-raw-sklearn-batcher-ee9be.18a900150ce1d637 namespace: kserve-ci-e2e-test resourceVersion: "13458" uid: 73de7dd1-3d8e-4570-8e1e-2c7c406626c2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:40:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-ee9be namespace: kserve-ci-e2e-test resourceVersion: "13435" uid: a0e25726-303a-4888-9eff-ebb378c4e8dd kind: Event lastTimestamp: "2026-04-23T13:40:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-ee9be": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:40:18Z" name: isvc-raw-sklearn-batcher-ee9be.18a900150ce220b8 namespace: kserve-ci-e2e-test resourceVersion: "13460" uid: 3491ec78-8955-4f02-b263-c844a5d9f862 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:41:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-ee9be namespace: kserve-ci-e2e-test resourceVersion: "14145" uid: a0e25726-303a-4888-9eff-ebb378c4e8dd kind: Event lastTimestamp: "2026-04-23T13:41:48Z" message: InferenceService [isvc-raw-sklearn-batcher-ee9be] is Ready metadata: creationTimestamp: "2026-04-23T13:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:41:48Z" name: isvc-raw-sklearn-batcher-ee9be.18a90029f92c930a namespace: kserve-ci-e2e-test resourceVersion: "14146" uid: 2384ff76-1eca-4bdd-89bd-7bad77a55056 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:51:09.949553Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-2a679f-predictor-555cd5b5f8-65tqj namespace: kserve-ci-e2e-test resourceVersion: "19091" uid: f4b89125-1e64-490b-9d91-2d7c4d9dcccf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-2a679f-predictor-555cd5b5f8-65tqj to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-04-23T13:51:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:51:09Z" name: isvc-secondary-2a679f-predictor-555cd5b5f8-65tqj.18a900acc12ae6b8 namespace: kserve-ci-e2e-test resourceVersion: "19095" uid: 5c5fcaf2-0afb-4893-bcbf-618c86dd1233 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-78b4f5d84d-ssxfd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-2a679f-predictor-555cd5b5f8-65tqj namespace: kserve-ci-e2e-test resourceVersion: "19099" uid: f4b89125-1e64-490b-9d91-2d7c4d9dcccf kind: Event lastTimestamp: "2026-04-23T13:51:10Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:51:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:51:10Z" name: isvc-secondary-2a679f-predictor-555cd5b5f8-65tqj.18a900acdb1ab59b namespace: kserve-ci-e2e-test resourceVersion: "19109" uid: fbae4da4-78e1-4ae7-aadf-f279fa0a7da9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:51:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-2a679f-predictor-555cd5b5f8-65tqj namespace: kserve-ci-e2e-test resourceVersion: "19093" uid: f4b89125-1e64-490b-9d91-2d7c4d9dcccf kind: Event lastTimestamp: "2026-04-23T13:51:14Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bfc8f044ee2a84e90b7b9876040a412b44d7b121a59a5fe7b22a946b15b61519" already present on machine metadata: creationTimestamp: "2026-04-23T13:51:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:14Z" name: isvc-secondary-2a679f-predictor-555cd5b5f8-65tqj.18a900acdc8bf6f5 namespace: kserve-ci-e2e-test resourceVersion: "19147" uid: 4ea2fd76-4403-4d54-ae1d-1cbc3a710714 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:51:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-2a679f-predictor-555cd5b5f8-65tqj namespace: kserve-ci-e2e-test resourceVersion: "19093" uid: f4b89125-1e64-490b-9d91-2d7c4d9dcccf kind: Event lastTimestamp: "2026-04-23T13:51:14Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T13:51:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:14Z" name: isvc-secondary-2a679f-predictor-555cd5b5f8-65tqj.18a900ace028cd70 namespace: kserve-ci-e2e-test resourceVersion: "19151" uid: ba589479-58ef-4ab5-8a58-bed44ddb1183 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:51:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-2a679f-predictor-555cd5b5f8-65tqj namespace: kserve-ci-e2e-test resourceVersion: "19093" uid: f4b89125-1e64-490b-9d91-2d7c4d9dcccf kind: Event lastTimestamp: "2026-04-23T13:51:14Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T13:51:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:14Z" name: isvc-secondary-2a679f-predictor-555cd5b5f8-65tqj.18a900ace0a78424 namespace: kserve-ci-e2e-test resourceVersion: "19152" uid: faa5dff4-f0d3-43bd-9bc1-c4f8d4aeedb5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-2a679f-predictor-555cd5b5f8-65tqj namespace: kserve-ci-e2e-test resourceVersion: "19093" uid: f4b89125-1e64-490b-9d91-2d7c4d9dcccf kind: Event lastTimestamp: "2026-04-23T13:51:21Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-2a679f-predictor-555cd5b5f8-65tqj_kserve-ci-e2e-test(f4b89125-1e64-490b-9d91-2d7c4d9dcccf) metadata: creationTimestamp: "2026-04-23T13:51:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:21Z" name: isvc-secondary-2a679f-predictor-555cd5b5f8-65tqj.18a900af6dab2075 namespace: kserve-ci-e2e-test resourceVersion: "19203" uid: 379ba246-3882-4517-a4f2-85e8ddf225cc reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-2a679f-predictor-555cd5b5f8 namespace: kserve-ci-e2e-test resourceVersion: "19082" uid: 07f04055-af13-4edf-b99f-3801d6867610 kind: Event lastTimestamp: "2026-04-23T13:51:09Z" message: 'Created pod: isvc-secondary-2a679f-predictor-555cd5b5f8-65tqj' metadata: creationTimestamp: "2026-04-23T13:51:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:51:09Z" name: isvc-secondary-2a679f-predictor-555cd5b5f8.18a900acc094cfe7 namespace: kserve-ci-e2e-test resourceVersion: "19092" uid: f7216806-af18-4968-bd4c-a78d76d16c31 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-2a679f-predictor namespace: kserve-ci-e2e-test resourceVersion: "19080" uid: 30f6aa84-a0ba-48d6-9155-252264b66595 kind: Event lastTimestamp: "2026-04-23T13:51:09Z" message: Scaled up replica set isvc-secondary-2a679f-predictor-555cd5b5f8 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:51:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:51:09Z" name: isvc-secondary-2a679f-predictor.18a900acbdbfd595 namespace: kserve-ci-e2e-test resourceVersion: "19084" uid: d2b3bfcb-4af4-425d-9486-c1ffab28a988 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-2a679f-predictor namespace: kserve-ci-e2e-test resourceVersion: "19081" uid: c2a8e097-b90a-4b65-9368-b3b001b5a923 kind: Event lastTimestamp: "2026-04-23T13:51:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:51:24Z" name: isvc-secondary-2a679f-predictor.18a900b03c4512b4 namespace: kserve-ci-e2e-test resourceVersion: "19235" uid: 8f1b2e1b-552a-4b43-9df8-82851d28107b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-2a679f-predictor namespace: kserve-ci-e2e-test resourceVersion: "19081" uid: c2a8e097-b90a-4b65-9368-b3b001b5a923 kind: Event lastTimestamp: "2026-04-23T13:51:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:51:24Z" name: isvc-secondary-2a679f-predictor.18a900b03cad494e namespace: kserve-ci-e2e-test resourceVersion: "19237" uid: 3216b58d-171c-4ebb-99e2-e3c63039ca43 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-23T13:51:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-2a679f namespace: kserve-ci-e2e-test resourceVersion: "19070" uid: 3caf2fcf-8eec-4952-8dd7-744d1fe0dc1e kind: Event lastTimestamp: "2026-04-23T13:51:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:51:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:51:21Z" name: isvc-secondary-2a679f.18a900acbd99da6a namespace: kserve-ci-e2e-test resourceVersion: "19208" uid: 8928395e-7350-4248-9e29-82585e80b2c9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:51:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-2a679f namespace: kserve-ci-e2e-test resourceVersion: "19070" uid: 3caf2fcf-8eec-4952-8dd7-744d1fe0dc1e kind: Event lastTimestamp: "2026-04-23T13:51:09Z" message: 'Failed to update status for InferenceService "isvc-secondary-2a679f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-2a679f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:51:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:51:09Z" name: isvc-secondary-2a679f.18a900acbdeaaf86 namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: 4a116848-515a-403f-8e98-483c55cbde3d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:51:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-2a679f namespace: kserve-ci-e2e-test resourceVersion: "19070" uid: 3caf2fcf-8eec-4952-8dd7-744d1fe0dc1e kind: Event lastTimestamp: "2026-04-23T13:51:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-2a679f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:51:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:51:09Z" name: isvc-secondary-2a679f.18a900acbdeb096c namespace: kserve-ci-e2e-test resourceVersion: "19102" uid: fa628db2-fb7d-4bb0-8078-d2b51b6bca35 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:42:03.560592Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 4f8f8aad-20d9-48c2-ab5d-741e0d08fa69 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-04-23T13:42:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:42:03Z" name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r.18a9002d89d8fb57 namespace: kserve-ci-e2e-test resourceVersion: "14322" uid: 1b388831-5294-4006-87d8-36037096527f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-78b4f5d84d-ssxfd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:03Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r namespace: kserve-ci-e2e-test resourceVersion: "14328" uid: 4f8f8aad-20d9-48c2-ab5d-741e0d08fa69 kind: Event lastTimestamp: "2026-04-23T13:42:03Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:42:03Z" name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r.18a9002da396575e namespace: kserve-ci-e2e-test resourceVersion: "14355" uid: e9c9934b-b2b3-49c4-9c29-a606d372d0df reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 4f8f8aad-20d9-48c2-ab5d-741e0d08fa69 kind: Event lastTimestamp: "2026-04-23T13:42:04Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bfc8f044ee2a84e90b7b9876040a412b44d7b121a59a5fe7b22a946b15b61519" already present on machine metadata: creationTimestamp: "2026-04-23T13:42:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:04Z" name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r.18a9002da557c8f8 namespace: kserve-ci-e2e-test resourceVersion: "14357" uid: 71cf30b9-07ab-4f05-801d-f70d506b0c1b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 4f8f8aad-20d9-48c2-ab5d-741e0d08fa69 kind: Event lastTimestamp: "2026-04-23T13:42:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T13:42:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:04Z" name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r.18a9002da9ad0acd namespace: kserve-ci-e2e-test resourceVersion: "14361" uid: 2d4f380b-09ca-415e-a947-7670d5a730e1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 4f8f8aad-20d9-48c2-ab5d-741e0d08fa69 kind: Event lastTimestamp: "2026-04-23T13:42:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T13:42:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:04Z" name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r.18a9002daa2cce5c namespace: kserve-ci-e2e-test resourceVersion: "14362" uid: 9ae03b3d-6b0f-4bf6-b260-ea8ac1eda698 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 4f8f8aad-20d9-48c2-ab5d-741e0d08fa69 kind: Event lastTimestamp: "2026-04-23T13:42:08Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T13:42:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:08Z" name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r.18a9002eaae5ba8c namespace: kserve-ci-e2e-test resourceVersion: "14407" uid: f1ce491e-fea2-4c0b-9846-101285fbbcd4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 4f8f8aad-20d9-48c2-ab5d-741e0d08fa69 kind: Event lastTimestamp: "2026-04-23T13:42:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:42:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:08Z" name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r.18a9002eaf95a720 namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: 6d2b5d58-8ade-4b79-a548-6529545199b5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 4f8f8aad-20d9-48c2-ab5d-741e0d08fa69 kind: Event lastTimestamp: "2026-04-23T13:42:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:42:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:08Z" name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r.18a9002eb01970cf namespace: kserve-ci-e2e-test resourceVersion: "14413" uid: fd29e65a-2a9b-44b8-85e0-9878e91b44bb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 4f8f8aad-20d9-48c2-ab5d-741e0d08fa69 kind: Event lastTimestamp: "2026-04-23T13:42:08Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:42:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:08Z" name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r.18a9002eb0278e08 namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: 0dfba58d-d42f-4b5e-a7e3-28c373a86a20 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 4f8f8aad-20d9-48c2-ab5d-741e0d08fa69 kind: Event lastTimestamp: "2026-04-23T13:42:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:42:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:08Z" name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r.18a9002eb36a0286 namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: 04e4029a-b927-4a0f-8147-4c875e5a4503 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 4f8f8aad-20d9-48c2-ab5d-741e0d08fa69 kind: Event lastTimestamp: "2026-04-23T13:42:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:42:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:08Z" name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r.18a9002eb3f1ac49 namespace: kserve-ci-e2e-test resourceVersion: "14420" uid: cf096aef-e260-4ba3-947a-76b855a983bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T13:42:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 4f8f8aad-20d9-48c2-ab5d-741e0d08fa69 kind: Event lastTimestamp: "2026-04-23T13:43:55Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:42:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:55Z" name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r.18a9002f230c1f5d namespace: kserve-ci-e2e-test resourceVersion: "15431" uid: 85f4c557-a7fb-4152-8b42-67806a2302bb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 4f8f8aad-20d9-48c2-ab5d-741e0d08fa69 kind: Event lastTimestamp: "2026-04-23T13:43:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:43:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:53Z" name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r.18a9004735afa642 namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: 09dd7c6e-a6fb-45f3-8f26-66e1bc7e59a3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 4f8f8aad-20d9-48c2-ab5d-741e0d08fa69 kind: Event lastTimestamp: "2026-04-23T13:43:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:43:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:53Z" name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r.18a9004735b023d6 namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: f6faec66-3732-473c-b2ca-59aa781e679a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: 4f8f8aad-20d9-48c2-ab5d-741e0d08fa69 kind: Event lastTimestamp: "2026-04-23T13:43:56Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:43:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:56Z" name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r.18a90047d13f6914 namespace: kserve-ci-e2e-test resourceVersion: "15446" uid: ee31f09b-8b2d-4a4b-b74e-0eaf54cb5e40 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: 3cfde7b5-aa4d-4cdb-b70c-09f8eb486d2c kind: Event lastTimestamp: "2026-04-23T13:42:03Z" message: 'Created pod: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd-ffg7r' metadata: creationTimestamp: "2026-04-23T13:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:42:03Z" name: isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd.18a9002d8917422f namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: b34821de-d062-435d-b2e9-52d20feacdae reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-b90a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14290" uid: 6bd438b1-abcc-4f29-922f-83e63eea878c kind: Event lastTimestamp: "2026-04-23T13:42:03Z" message: Scaled up replica set isvc-sklearn-graph-raw-b90a6-predictor-67c897bfbd from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:42:03Z" name: isvc-sklearn-graph-raw-b90a6-predictor.18a9002d8436f41e namespace: kserve-ci-e2e-test resourceVersion: "14294" uid: bb7d19a9-32f0-4041-9592-ba921883d074 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:42:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-b90a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14291" uid: 4727fb92-cc55-47de-8d16-01d179f31581 kind: Event lastTimestamp: "2026-04-23T13:42:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:42:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:42:33Z" name: isvc-sklearn-graph-raw-b90a6-predictor.18a9003103030ac6 namespace: kserve-ci-e2e-test resourceVersion: "14631" uid: b167933e-d945-4b6f-bcb0-919f7d941430 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:42:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-b90a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14291" uid: 4727fb92-cc55-47de-8d16-01d179f31581 kind: Event lastTimestamp: "2026-04-23T13:42:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:42:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:42:33Z" name: isvc-sklearn-graph-raw-b90a6-predictor.18a9003103688853 namespace: kserve-ci-e2e-test resourceVersion: "14632" uid: 726ad1d7-570b-422f-8ad5-d88228535786 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T13:42:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-b90a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14511" uid: 4727fb92-cc55-47de-8d16-01d179f31581 kind: Event lastTimestamp: "2026-04-23T13:43:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:42:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:48Z" name: isvc-sklearn-graph-raw-b90a6-predictor.18a9003801722e55 namespace: kserve-ci-e2e-test resourceVersion: "15233" uid: d7df8911-3473-4659-9e15-8c2d3e46cfec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-23T13:42:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-b90a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14511" uid: 4727fb92-cc55-47de-8d16-01d179f31581 kind: Event lastTimestamp: "2026-04-23T13:43:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:42:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:48Z" name: isvc-sklearn-graph-raw-b90a6-predictor.18a9003801dacd51 namespace: kserve-ci-e2e-test resourceVersion: "15234" uid: fd2babb5-57b5-4871-841e-59502f3629fe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T13:42:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-b90a6 namespace: kserve-ci-e2e-test resourceVersion: "14262" uid: 37e2c3d6-6cff-48cb-b09c-446e78f5680a kind: Event lastTimestamp: "2026-04-23T13:43:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:25Z" name: isvc-sklearn-graph-raw-b90a6.18a9002d8411fe1d namespace: kserve-ci-e2e-test resourceVersion: "15026" uid: 4d213ccf-f3c8-4ca0-bca4-2d1e051febb7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-b90a6 namespace: kserve-ci-e2e-test resourceVersion: "14262" uid: 37e2c3d6-6cff-48cb-b09c-446e78f5680a kind: Event lastTimestamp: "2026-04-23T13:42:03Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-b90a6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-b90a6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:42:03Z" name: isvc-sklearn-graph-raw-b90a6.18a9002d848a91cc namespace: kserve-ci-e2e-test resourceVersion: "14297" uid: 7636c114-2969-4d48-babf-c5556217deba reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-b90a6 namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: 37e2c3d6-6cff-48cb-b09c-446e78f5680a kind: Event lastTimestamp: "2026-04-23T13:43:25Z" message: InferenceService [isvc-sklearn-graph-raw-b90a6] is Ready metadata: creationTimestamp: "2026-04-23T13:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:25Z" name: isvc-sklearn-graph-raw-b90a6.18a900409d7202da namespace: kserve-ci-e2e-test resourceVersion: "15025" uid: d9b64db6-b6a8-4379-8439-07adbb06dd71 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:43:53.894744Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: 8601b18e-0763-4e85-be49-f777e7b19cb1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-04-23T13:43:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:43:53Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm.18a900473a46940f namespace: kserve-ci-e2e-test resourceVersion: "15354" uid: 35baefe8-f529-44b9-8f64-61ce8036b745 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-78b4f5d84d-ssxfd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:54Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: 8601b18e-0763-4e85-be49-f777e7b19cb1 kind: Event lastTimestamp: "2026-04-23T13:43:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-hpa-4f8db-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T13:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:54Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm.18a90047463a4f08 namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: 08c0a8ad-5744-450f-94c9-246e3128ff40 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:54Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm namespace: kserve-ci-e2e-test resourceVersion: "15366" uid: 8601b18e-0763-4e85-be49-f777e7b19cb1 kind: Event lastTimestamp: "2026-04-23T13:43:54Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:43:54Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm.18a9004777dc76c4 namespace: kserve-ci-e2e-test resourceVersion: "15422" uid: c7fa888b-cc3f-4200-85b1-f9d0e497ae22 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: 8601b18e-0763-4e85-be49-f777e7b19cb1 kind: Event lastTimestamp: "2026-04-23T13:43:54Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bfc8f044ee2a84e90b7b9876040a412b44d7b121a59a5fe7b22a946b15b61519" already present on machine metadata: creationTimestamp: "2026-04-23T13:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:54Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm.18a900477932ce91 namespace: kserve-ci-e2e-test resourceVersion: "15424" uid: 9585c2bf-da6b-4a83-868b-5c0df5b55d93 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: 8601b18e-0763-4e85-be49-f777e7b19cb1 kind: Event lastTimestamp: "2026-04-23T13:43:55Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T13:43:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:55Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm.18a900477ca6e38e namespace: kserve-ci-e2e-test resourceVersion: "15425" uid: ddb84d72-b84e-400e-b9a8-d7728656d209 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: 8601b18e-0763-4e85-be49-f777e7b19cb1 kind: Event lastTimestamp: "2026-04-23T13:43:55Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T13:43:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:55Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm.18a900477d269996 namespace: kserve-ci-e2e-test resourceVersion: "15426" uid: 7b57d745-ecd6-4b6f-812e-19af635f4a21 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: 8601b18e-0763-4e85-be49-f777e7b19cb1 kind: Event lastTimestamp: "2026-04-23T13:43:58Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T13:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:58Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm.18a900485f7a959a namespace: kserve-ci-e2e-test resourceVersion: "15465" uid: 04ed7ad4-fb34-49a3-be75-4ac95573a283 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: 8601b18e-0763-4e85-be49-f777e7b19cb1 kind: Event lastTimestamp: "2026-04-23T13:43:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:58Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm.18a9004863714bac namespace: kserve-ci-e2e-test resourceVersion: "15475" uid: 53b92bf9-7fbf-4e1b-a820-2e9ba1854924 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: 8601b18e-0763-4e85-be49-f777e7b19cb1 kind: Event lastTimestamp: "2026-04-23T13:43:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:58Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm.18a9004864358b4d namespace: kserve-ci-e2e-test resourceVersion: "15478" uid: 44f92739-9142-44cf-87b8-d6b443ee1260 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: 8601b18e-0763-4e85-be49-f777e7b19cb1 kind: Event lastTimestamp: "2026-04-23T13:43:58Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:58Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm.18a900486445da1b namespace: kserve-ci-e2e-test resourceVersion: "15481" uid: d212f4e2-0b53-4842-a238-d87b95c1ea98 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: 8601b18e-0763-4e85-be49-f777e7b19cb1 kind: Event lastTimestamp: "2026-04-23T13:43:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:58Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm.18a9004867b38f31 namespace: kserve-ci-e2e-test resourceVersion: "15484" uid: e0e73a3f-0720-4c02-8179-12dc36e612ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: 8601b18e-0763-4e85-be49-f777e7b19cb1 kind: Event lastTimestamp: "2026-04-23T13:43:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:58Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm.18a900486842a807 namespace: kserve-ci-e2e-test resourceVersion: "15487" uid: 3e3d2288-a7de-47f7-a347-fd0e4e5ab948 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T13:44:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: 8601b18e-0763-4e85-be49-f777e7b19cb1 kind: Event lastTimestamp: "2026-04-23T13:45:36Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:44:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:36Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm.18a90048d77dc390 namespace: kserve-ci-e2e-test resourceVersion: "16389" uid: d1839a8a-9d2c-4739-9231-605ee927058e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: 8601b18e-0763-4e85-be49-f777e7b19cb1 kind: Event lastTimestamp: "2026-04-23T13:45:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:34Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm.18a9005e96ffc847 namespace: kserve-ci-e2e-test resourceVersion: "16315" uid: aa479d21-702b-459e-ab11-0981de832d55 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: 8601b18e-0763-4e85-be49-f777e7b19cb1 kind: Event lastTimestamp: "2026-04-23T13:45:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:34Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm.18a9005e97005828 namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: d2cfef52-c836-4d6d-b3f5-15f1de3a6ed3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm namespace: kserve-ci-e2e-test resourceVersion: "15350" uid: 8601b18e-0763-4e85-be49-f777e7b19cb1 kind: Event lastTimestamp: "2026-04-23T13:45:36Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:36Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm.18a9005f31b58184 namespace: kserve-ci-e2e-test resourceVersion: "16387" uid: e1c739a1-cf41-4277-b72e-5b1ea16c0225 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568 namespace: kserve-ci-e2e-test resourceVersion: "15317" uid: b430f10f-8a46-41fb-b192-ea2a6cf10d0b kind: Event lastTimestamp: "2026-04-23T13:43:53Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568-f2srm' metadata: creationTimestamp: "2026-04-23T13:43:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:53Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568.18a9004739ce2dbf namespace: kserve-ci-e2e-test resourceVersion: "15348" uid: 43284014-4952-4e8a-bbb4-9c8338179f8a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-4f8db-predictor namespace: kserve-ci-e2e-test resourceVersion: "15312" uid: 70af29b8-f43b-411f-9aaf-9f2f736a2ffb kind: Event lastTimestamp: "2026-04-23T13:43:53Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-4f8db-predictor-75858f568 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:43:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:53Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor.18a900473411f877 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 9ab9570c-4916-4a5f-b72e-6feda3bb5a70 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:44:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-4f8db-predictor namespace: kserve-ci-e2e-test resourceVersion: "15316" uid: e7a01fa6-8b7f-4066-be7f-0790f087c0e0 kind: Event lastTimestamp: "2026-04-23T13:44:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:44:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:53Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor.18a9004ab2c3d2e2 namespace: kserve-ci-e2e-test resourceVersion: "15914" uid: bd921e37-7b6a-40d6-a7d0-239080e78def reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:44:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-4f8db-predictor namespace: kserve-ci-e2e-test resourceVersion: "15316" uid: e7a01fa6-8b7f-4066-be7f-0790f087c0e0 kind: Event lastTimestamp: "2026-04-23T13:44:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:44:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:53Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor.18a9004ab329ceb0 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: 46aa9ab8-52fa-4457-9bae-a0699dbcd551 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:45:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-4f8db-predictor namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: e7a01fa6-8b7f-4066-be7f-0790f087c0e0 kind: Event lastTimestamp: "2026-04-23T13:45:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:45:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:23Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor.18a90058af18c3c8 namespace: kserve-ci-e2e-test resourceVersion: "16156" uid: 42f0f760-2b62-4905-87d8-5e1cc7517935 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:45:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-4f8db-predictor namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: e7a01fa6-8b7f-4066-be7f-0790f087c0e0 kind: Event lastTimestamp: "2026-04-23T13:45:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:45:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:23Z" name: isvc-sklearn-graph-raw-hpa-4f8db-predictor.18a90058af7e89c3 namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: cc891d9b-e263-4fe2-bc0e-9ef815adbebb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T13:43:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-4f8db namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: a0b97c2b-2d88-42cd-9918-867cebd4e7b0 kind: Event lastTimestamp: "2026-04-23T13:45:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:43:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:45:06Z" name: isvc-sklearn-graph-raw-hpa-4f8db.18a9004733c4963b namespace: kserve-ci-e2e-test resourceVersion: "16023" uid: 00eb4604-aec3-4912-9170-53b197deb84d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-4f8db namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: a0b97c2b-2d88-42cd-9918-867cebd4e7b0 kind: Event lastTimestamp: "2026-04-23T13:43:53Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-4f8db": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-4f8db": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:43:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:53Z" name: isvc-sklearn-graph-raw-hpa-4f8db.18a90047349b370a namespace: kserve-ci-e2e-test resourceVersion: "15327" uid: e9472077-673b-44bc-9a7a-a392372d66f4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-4f8db namespace: kserve-ci-e2e-test resourceVersion: "16018" uid: a0b97c2b-2d88-42cd-9918-867cebd4e7b0 kind: Event lastTimestamp: "2026-04-23T13:45:06Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-4f8db] is Ready metadata: creationTimestamp: "2026-04-23T13:45:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:45:06Z" name: isvc-sklearn-graph-raw-hpa-4f8db.18a900583ab8f10e namespace: kserve-ci-e2e-test resourceVersion: "16019" uid: 387f1fb3-8204-4775-95ea-a4e8ca3ac604 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:47:09.556785Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd namespace: kserve-ci-e2e-test resourceVersion: "17197" uid: 9b7b19d9-effb-4ea6-bbf6-e3a0b9f4b069 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-04-23T13:47:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:47:09Z" name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd.18a90074c8a45358 namespace: kserve-ci-e2e-test resourceVersion: "17209" uid: bc934afd-3a55-40a4-8b55-701157ff5e26 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-78b4f5d84d-ssxfd source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:47:09Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 9b7b19d9-effb-4ea6-bbf6-e3a0b9f4b069 kind: Event lastTimestamp: "2026-04-23T13:47:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-7affb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T13:47:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:47:10Z" name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd.18a90074d1f03fe4 namespace: kserve-ci-e2e-test resourceVersion: "17230" uid: c62da2e3-6117-4efb-90d2-34cb010622e2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:47:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd namespace: kserve-ci-e2e-test resourceVersion: "17217" uid: 9b7b19d9-effb-4ea6-bbf6-e3a0b9f4b069 kind: Event lastTimestamp: "2026-04-23T13:47:11Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:47:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:47:11Z" name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd.18a9007540d0b3a3 namespace: kserve-ci-e2e-test resourceVersion: "17247" uid: c97c0ee8-014f-4cf0-8d1e-f5a91b4f805d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:47:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 9b7b19d9-effb-4ea6-bbf6-e3a0b9f4b069 kind: Event lastTimestamp: "2026-04-23T13:47:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bfc8f044ee2a84e90b7b9876040a412b44d7b121a59a5fe7b22a946b15b61519" already present on machine metadata: creationTimestamp: "2026-04-23T13:47:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:47:11Z" name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd.18a900754225b5f1 namespace: kserve-ci-e2e-test resourceVersion: "17249" uid: b3992dbd-b897-46fb-af43-7616e467ccf6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:47:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 9b7b19d9-effb-4ea6-bbf6-e3a0b9f4b069 kind: Event lastTimestamp: "2026-04-23T13:47:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T13:47:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:47:11Z" name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd.18a9007545a750a3 namespace: kserve-ci-e2e-test resourceVersion: "17251" uid: 5f8c1232-6ac4-4f02-9f0c-7832ee464358 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:47:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 9b7b19d9-effb-4ea6-bbf6-e3a0b9f4b069 kind: Event lastTimestamp: "2026-04-23T13:47:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T13:47:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:47:11Z" name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd.18a9007546266770 namespace: kserve-ci-e2e-test resourceVersion: "17252" uid: 67871d3d-7b20-4ce4-995a-f05c0034b963 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:47:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 9b7b19d9-effb-4ea6-bbf6-e3a0b9f4b069 kind: Event lastTimestamp: "2026-04-23T13:47:16Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T13:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:47:16Z" name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd.18a9007668d41664 namespace: kserve-ci-e2e-test resourceVersion: "17294" uid: ebf45a35-b8bb-4947-b302-b207373b38da reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:47:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 9b7b19d9-effb-4ea6-bbf6-e3a0b9f4b069 kind: Event lastTimestamp: "2026-04-23T13:47:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:47:16Z" name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd.18a900766c19a2ec namespace: kserve-ci-e2e-test resourceVersion: "17298" uid: bc753f17-2266-49ce-8287-22e672a4ab7a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:47:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 9b7b19d9-effb-4ea6-bbf6-e3a0b9f4b069 kind: Event lastTimestamp: "2026-04-23T13:47:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:47:16Z" name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd.18a900766c9585ca namespace: kserve-ci-e2e-test resourceVersion: "17299" uid: fa1ce7fb-4b2d-41ec-8139-bcca7b363b3f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:47:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 9b7b19d9-effb-4ea6-bbf6-e3a0b9f4b069 kind: Event lastTimestamp: "2026-04-23T13:47:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:47:16Z" name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd.18a900766ca2ad97 namespace: kserve-ci-e2e-test resourceVersion: "17301" uid: 112be600-8957-4f7a-b3ff-e290a546ff96 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:47:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 9b7b19d9-effb-4ea6-bbf6-e3a0b9f4b069 kind: Event lastTimestamp: "2026-04-23T13:47:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:47:16Z" name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd.18a900766fba65da namespace: kserve-ci-e2e-test resourceVersion: "17303" uid: b5f255fb-5ece-4032-8660-8ea74f553a65 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:47:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 9b7b19d9-effb-4ea6-bbf6-e3a0b9f4b069 kind: Event lastTimestamp: "2026-04-23T13:47:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:47:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:47:16Z" name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd.18a900767038cdaa namespace: kserve-ci-e2e-test resourceVersion: "17304" uid: 3d535772-7b42-444a-b000-779882a1e7d7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 17 eventTime: null firstTimestamp: "2026-04-23T13:47:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 9b7b19d9-effb-4ea6-bbf6-e3a0b9f4b069 kind: Event lastTimestamp: "2026-04-23T13:49:50Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:49:50Z" name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd.18a90076e0b66517 namespace: kserve-ci-e2e-test resourceVersion: "18464" uid: e4836bdb-3fe9-4aca-a7b9-8a644b9e1e76 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:49:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 9b7b19d9-effb-4ea6-bbf6-e3a0b9f4b069 kind: Event lastTimestamp: "2026-04-23T13:49:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:49:49Z" name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd.18a9009a134eb1c2 namespace: kserve-ci-e2e-test resourceVersion: "18431" uid: f69c9017-7e93-4198-a98f-10af55e6795b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:49:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 9b7b19d9-effb-4ea6-bbf6-e3a0b9f4b069 kind: Event lastTimestamp: "2026-04-23T13:49:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:49:49Z" name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd.18a9009a134f626a namespace: kserve-ci-e2e-test resourceVersion: "18433" uid: c92dc2eb-1ca6-4574-8d9a-34a4cc06ebfe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:49:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 9b7b19d9-effb-4ea6-bbf6-e3a0b9f4b069 kind: Event lastTimestamp: "2026-04-23T13:49:54Z" message: 'Readiness probe failed: Get "https://10.134.0.40:8643/healthz": dial tcp 10.134.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:49:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:49:54Z" name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd.18a9009b33326f5c namespace: kserve-ci-e2e-test resourceVersion: "18506" uid: b95146e7-3ecb-4253-8995-43a48893d428 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:47:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db namespace: kserve-ci-e2e-test resourceVersion: "17181" uid: b0da785f-6e12-4fe1-9b48-d638fe4705e2 kind: Event lastTimestamp: "2026-04-23T13:47:09Z" message: 'Created pod: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db-s6fkd' metadata: creationTimestamp: "2026-04-23T13:47:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:47:09Z" name: isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db.18a90074c7dcde59 namespace: kserve-ci-e2e-test resourceVersion: "17199" uid: e0a5bfab-4634-4341-bf8f-ac49d5c08307 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:47:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-7affb-predictor namespace: kserve-ci-e2e-test resourceVersion: "17177" uid: f519765f-f0ab-4736-a11d-ea6d28194f45 kind: Event lastTimestamp: "2026-04-23T13:47:09Z" message: Scaled up replica set isvc-sklearn-scale-raw-7affb-predictor-745ccbf6db from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:47:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:47:09Z" name: isvc-sklearn-scale-raw-7affb-predictor.18a90074c4bfd815 namespace: kserve-ci-e2e-test resourceVersion: "17184" uid: c0276b2d-7ac8-4a12-bd38-05595461a092 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:47:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-7affb-predictor namespace: kserve-ci-e2e-test resourceVersion: "17179" uid: fd7138e2-b017-457f-9a83-e23450498540 kind: Event lastTimestamp: "2026-04-23T13:47:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:47:54Z" name: isvc-sklearn-scale-raw-7affb-predictor.18a900784331588b namespace: kserve-ci-e2e-test resourceVersion: "17586" uid: 4c6a638f-c870-4189-900f-f8e74170598a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:47:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-7affb-predictor namespace: kserve-ci-e2e-test resourceVersion: "17179" uid: fd7138e2-b017-457f-9a83-e23450498540 kind: Event lastTimestamp: "2026-04-23T13:47:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:47:54Z" name: isvc-sklearn-scale-raw-7affb-predictor.18a900784394ab84 namespace: kserve-ci-e2e-test resourceVersion: "17587" uid: 2befe20b-fe25-42ae-895a-fa0ffa96a51c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-23T13:47:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-7affb-predictor namespace: kserve-ci-e2e-test resourceVersion: "17367" uid: fd7138e2-b017-457f-9a83-e23450498540 kind: Event lastTimestamp: "2026-04-23T13:49:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:47:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:49:39Z" name: isvc-sklearn-scale-raw-7affb-predictor.18a9007bc2b45a7a namespace: kserve-ci-e2e-test resourceVersion: "18324" uid: 95a07a3d-fa7c-43cd-ac0e-4ea36f722bca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-23T13:47:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-7affb-predictor namespace: kserve-ci-e2e-test resourceVersion: "17367" uid: fd7138e2-b017-457f-9a83-e23450498540 kind: Event lastTimestamp: "2026-04-23T13:49:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:47:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:49:39Z" name: isvc-sklearn-scale-raw-7affb-predictor.18a9007bc321d87e namespace: kserve-ci-e2e-test resourceVersion: "18325" uid: c05eae58-6823-45e3-a0c4-7f66bfe781ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 14 eventTime: null firstTimestamp: "2026-04-23T13:47:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-7affb namespace: kserve-ci-e2e-test resourceVersion: "17151" uid: 8600ea11-7413-4da6-8e80-7608cd6a41b8 kind: Event lastTimestamp: "2026-04-23T13:49:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:47:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:49:40Z" name: isvc-sklearn-scale-raw-7affb.18a90074c49ebabb namespace: kserve-ci-e2e-test resourceVersion: "18341" uid: 0668c29a-c81b-436d-81c0-2327f62520b5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:47:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-7affb namespace: kserve-ci-e2e-test resourceVersion: "17151" uid: 8600ea11-7413-4da6-8e80-7608cd6a41b8 kind: Event lastTimestamp: "2026-04-23T13:47:09Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-7affb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-7affb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:47:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:47:09Z" name: isvc-sklearn-scale-raw-7affb.18a90074c523dacc namespace: kserve-ci-e2e-test resourceVersion: "17221" uid: f2eb0541-c4a9-41ed-9d8d-339cb461c057 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:47:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-7affb namespace: kserve-ci-e2e-test resourceVersion: "17163" uid: 8600ea11-7413-4da6-8e80-7608cd6a41b8 kind: Event lastTimestamp: "2026-04-23T13:47:09Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-7affb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:47:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:47:09Z" name: isvc-sklearn-scale-raw-7affb.18a90074cb9533f9 namespace: kserve-ci-e2e-test resourceVersion: "17222" uid: 318475d0-a672-417c-a408-d4597f86a25a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:49:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-7affb namespace: kserve-ci-e2e-test resourceVersion: "18339" uid: 8600ea11-7413-4da6-8e80-7608cd6a41b8 kind: Event lastTimestamp: "2026-04-23T13:49:40Z" message: InferenceService [isvc-sklearn-scale-raw-7affb] is Ready metadata: creationTimestamp: "2026-04-23T13:49:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:49:40Z" name: isvc-sklearn-scale-raw-7affb.18a90097ee7a6127 namespace: kserve-ci-e2e-test resourceVersion: "18340" uid: 96a87a83-a01f-4bd2-85d6-96b34b2d2702 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:42:03.628709Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98 namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: a9ba9358-4436-47f4-9437-5acc89fd41c9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98 to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-04-23T13:42:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:42:03Z" name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98.18a9002d8de863ae namespace: kserve-ci-e2e-test resourceVersion: "14340" uid: e7363df2-d1ac-4908-98c1-6b8b87b6a341 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-78b4f5d84d-ssxfd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98 namespace: kserve-ci-e2e-test resourceVersion: "14347" uid: a9ba9358-4436-47f4-9437-5acc89fd41c9 kind: Event lastTimestamp: "2026-04-23T13:42:04Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:42:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:42:04Z" name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98.18a9002da839034a namespace: kserve-ci-e2e-test resourceVersion: "14358" uid: a74bd5ba-6da9-4402-b320-5c332b65b1c4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: a9ba9358-4436-47f4-9437-5acc89fd41c9 kind: Event lastTimestamp: "2026-04-23T13:42:04Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bfc8f044ee2a84e90b7b9876040a412b44d7b121a59a5fe7b22a946b15b61519" already present on machine metadata: creationTimestamp: "2026-04-23T13:42:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:04Z" name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98.18a9002da9a73329 namespace: kserve-ci-e2e-test resourceVersion: "14360" uid: 4227a0ac-dd45-4ea0-81fc-81586c7199ca reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: a9ba9358-4436-47f4-9437-5acc89fd41c9 kind: Event lastTimestamp: "2026-04-23T13:42:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T13:42:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:04Z" name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98.18a9002dac759d3f namespace: kserve-ci-e2e-test resourceVersion: "14363" uid: 150c6f4b-32b1-4f84-8e35-1e4bdc9376f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: a9ba9358-4436-47f4-9437-5acc89fd41c9 kind: Event lastTimestamp: "2026-04-23T13:42:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T13:42:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:04Z" name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98.18a9002dad369444 namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: fc13e057-7c31-4f0d-bdfe-3d60d28db49c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: a9ba9358-4436-47f4-9437-5acc89fd41c9 kind: Event lastTimestamp: "2026-04-23T13:42:08Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-23T13:42:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:08Z" name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98.18a9002eaad7282e namespace: kserve-ci-e2e-test resourceVersion: "14406" uid: 9006223f-b6d4-4e50-ae36-858f14a27866 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: a9ba9358-4436-47f4-9437-5acc89fd41c9 kind: Event lastTimestamp: "2026-04-23T13:42:27Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 18.623s (18.623s including waiting). Image size: 1306414326 bytes.' metadata: creationTimestamp: "2026-04-23T13:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:27Z" name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98.18a9003300df0945 namespace: kserve-ci-e2e-test resourceVersion: "14575" uid: fed02c10-5a1b-490e-8bc9-c847ef8a0dc1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: a9ba9358-4436-47f4-9437-5acc89fd41c9 kind: Event lastTimestamp: "2026-04-23T13:42:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:27Z" name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98.18a900330415c4ff namespace: kserve-ci-e2e-test resourceVersion: "14577" uid: 64fd3d08-a290-48c6-8deb-34e609e69397 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: a9ba9358-4436-47f4-9437-5acc89fd41c9 kind: Event lastTimestamp: "2026-04-23T13:42:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:27Z" name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98.18a9003304a3f440 namespace: kserve-ci-e2e-test resourceVersion: "14578" uid: 3655dc2c-38fd-4960-bd10-d5fb4929756d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: a9ba9358-4436-47f4-9437-5acc89fd41c9 kind: Event lastTimestamp: "2026-04-23T13:42:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:27Z" name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98.18a9003304b0f335 namespace: kserve-ci-e2e-test resourceVersion: "14579" uid: 34d11b3a-751f-48e7-a26c-67450cab7afc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: a9ba9358-4436-47f4-9437-5acc89fd41c9 kind: Event lastTimestamp: "2026-04-23T13:42:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:27Z" name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98.18a90033080c4b8b namespace: kserve-ci-e2e-test resourceVersion: "14581" uid: 44b26908-742c-47c3-aeac-83ebfa510d21 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: a9ba9358-4436-47f4-9437-5acc89fd41c9 kind: Event lastTimestamp: "2026-04-23T13:42:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:42:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:42:27Z" name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98.18a900330888c860 namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: fed4e875-ea32-4f76-bc0d-a31b3b965b67 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T13:42:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: a9ba9358-4436-47f4-9437-5acc89fd41c9 kind: Event lastTimestamp: "2026-04-23T13:43:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:42:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:54Z" name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98.18a9003357efbbbd namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: bfa18f57-d908-43d9-8612-1adaf6912943 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: a9ba9358-4436-47f4-9437-5acc89fd41c9 kind: Event lastTimestamp: "2026-04-23T13:43:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:43:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:53Z" name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98.18a900473e0a43ed namespace: kserve-ci-e2e-test resourceVersion: "15383" uid: 59248fcb-1f56-4d31-9ed6-7ac1312f8ed4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: a9ba9358-4436-47f4-9437-5acc89fd41c9 kind: Event lastTimestamp: "2026-04-23T13:43:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:43:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:53Z" name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98.18a900473e0af7a5 namespace: kserve-ci-e2e-test resourceVersion: "15385" uid: ae56ba7d-de5a-4912-95bb-d229d07b9366 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: a9ba9358-4436-47f4-9437-5acc89fd41c9 kind: Event lastTimestamp: "2026-04-23T13:43:54Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:54Z" name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98.18a900475e1499e8 namespace: kserve-ci-e2e-test resourceVersion: "15409" uid: c27e94bc-75ce-4984-ab10-2aca8f700edc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789 namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: 70533e70-6e89-4bac-ae2a-603f6fa70117 kind: Event lastTimestamp: "2026-04-23T13:42:03Z" message: 'Created pod: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789-8tb98' metadata: creationTimestamp: "2026-04-23T13:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:42:03Z" name: isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789.18a9002d8d53b345 namespace: kserve-ci-e2e-test resourceVersion: "14336" uid: 4fbf6826-830c-4938-81b0-7472ed1215ef reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:42:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-b90a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14312" uid: 62e5d484-65b3-43be-88fb-8d15d88ec6ba kind: Event lastTimestamp: "2026-04-23T13:42:03Z" message: Scaled up replica set isvc-xgboost-graph-raw-b90a6-predictor-6bbb968789 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:42:03Z" name: isvc-xgboost-graph-raw-b90a6-predictor.18a9002d89dceeda namespace: kserve-ci-e2e-test resourceVersion: "14319" uid: 1fe7d3d8-3eda-4a89-b2ec-23b98b138887 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:42:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-b90a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 6a26b095-3ea3-465f-9b02-ba96fc4ec259 kind: Event lastTimestamp: "2026-04-23T13:43:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:42:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:03Z" name: isvc-xgboost-graph-raw-b90a6-predictor.18a90031093f8313 namespace: kserve-ci-e2e-test resourceVersion: "14854" uid: 567416e8-03a4-4b52-9fd7-f0551c9e3259 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:42:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-b90a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14318" uid: 6a26b095-3ea3-465f-9b02-ba96fc4ec259 kind: Event lastTimestamp: "2026-04-23T13:43:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:42:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:03Z" name: isvc-xgboost-graph-raw-b90a6-predictor.18a900310a73136f namespace: kserve-ci-e2e-test resourceVersion: "14855" uid: 5193415b-3152-4f46-8d1a-d66f9e254ca8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:43:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-b90a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14516" uid: 6a26b095-3ea3-465f-9b02-ba96fc4ec259 kind: Event lastTimestamp: "2026-04-23T13:43:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:48Z" name: isvc-xgboost-graph-raw-b90a6-predictor.18a9003f056d00a1 namespace: kserve-ci-e2e-test resourceVersion: "15236" uid: f5203762-e7ab-41c1-8608-d0a531372584 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-23T13:43:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-b90a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14516" uid: 6a26b095-3ea3-465f-9b02-ba96fc4ec259 kind: Event lastTimestamp: "2026-04-23T13:43:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:48Z" name: isvc-xgboost-graph-raw-b90a6-predictor.18a9003f05cef24d namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: 8f49eb23-533a-470e-b8e7-e999970b1192 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-23T13:42:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-b90a6 namespace: kserve-ci-e2e-test resourceVersion: "14303" uid: 386595f2-dee2-4ded-af9e-a509c4c3d198 kind: Event lastTimestamp: "2026-04-23T13:43:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:34Z" name: isvc-xgboost-graph-raw-b90a6.18a9002d89cc41a5 namespace: kserve-ci-e2e-test resourceVersion: "15102" uid: e1799233-ed8f-43a0-a6d6-6eb2c343b427 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-b90a6 namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: 386595f2-dee2-4ded-af9e-a509c4c3d198 kind: Event lastTimestamp: "2026-04-23T13:43:34Z" message: InferenceService [isvc-xgboost-graph-raw-b90a6] is Ready metadata: creationTimestamp: "2026-04-23T13:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:34Z" name: isvc-xgboost-graph-raw-b90a6.18a90042bbcd51d4 namespace: kserve-ci-e2e-test resourceVersion: "15101" uid: dd3454e4-6314-49f0-8f66-84b8f2e4135f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:43:53.976859Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6 namespace: kserve-ci-e2e-test resourceVersion: "15384" uid: b802602d-f877-40b1-9a89-e26ec27b66ca kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6 to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-04-23T13:43:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:43:53Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6.18a900473f2b9d92 namespace: kserve-ci-e2e-test resourceVersion: "15389" uid: c8741d96-fd82-4549-98f7-999f478044ef reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-78b4f5d84d-ssxfd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:54Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6 namespace: kserve-ci-e2e-test resourceVersion: "15394" uid: b802602d-f877-40b1-9a89-e26ec27b66ca kind: Event lastTimestamp: "2026-04-23T13:43:54Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:43:54Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6.18a90047655bf926 namespace: kserve-ci-e2e-test resourceVersion: "15411" uid: 21939f77-7575-4e31-8570-f3b92e634b49 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: b802602d-f877-40b1-9a89-e26ec27b66ca kind: Event lastTimestamp: "2026-04-23T13:43:54Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bfc8f044ee2a84e90b7b9876040a412b44d7b121a59a5fe7b22a946b15b61519" already present on machine metadata: creationTimestamp: "2026-04-23T13:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:54Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6.18a9004766c5bbeb namespace: kserve-ci-e2e-test resourceVersion: "15413" uid: 961e807c-6d69-4681-929a-639167893689 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: b802602d-f877-40b1-9a89-e26ec27b66ca kind: Event lastTimestamp: "2026-04-23T13:43:54Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T13:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:54Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6.18a90047695669fe namespace: kserve-ci-e2e-test resourceVersion: "15415" uid: 87b14d8f-25f3-44b2-a098-5011e3989207 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: b802602d-f877-40b1-9a89-e26ec27b66ca kind: Event lastTimestamp: "2026-04-23T13:43:54Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T13:43:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:54Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6.18a9004769cf7e96 namespace: kserve-ci-e2e-test resourceVersion: "15416" uid: ffdd837a-3b64-41e1-a6cf-8cbfefcbcba4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: b802602d-f877-40b1-9a89-e26ec27b66ca kind: Event lastTimestamp: "2026-04-23T13:43:58Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-23T13:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:58Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6.18a900485f951220 namespace: kserve-ci-e2e-test resourceVersion: "15466" uid: 74176878-ea46-4aa0-84fe-4f6aefac60af reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: b802602d-f877-40b1-9a89-e26ec27b66ca kind: Event lastTimestamp: "2026-04-23T13:43:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:58Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6.18a90048638b846e namespace: kserve-ci-e2e-test resourceVersion: "15477" uid: 79aeea89-6ac2-4dcf-88a3-c1cfd74a614a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: b802602d-f877-40b1-9a89-e26ec27b66ca kind: Event lastTimestamp: "2026-04-23T13:43:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:58Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6.18a9004864358b50 namespace: kserve-ci-e2e-test resourceVersion: "15479" uid: 011c9b6b-17a8-410e-a80e-ca220ef2210f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: b802602d-f877-40b1-9a89-e26ec27b66ca kind: Event lastTimestamp: "2026-04-23T13:43:58Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:58Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6.18a900486444fb9d namespace: kserve-ci-e2e-test resourceVersion: "15480" uid: f7ad01d9-0b52-4bba-9ba9-48add345a6c1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: b802602d-f877-40b1-9a89-e26ec27b66ca kind: Event lastTimestamp: "2026-04-23T13:43:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:58Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6.18a9004867c8891a namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: 2a944f43-40b2-4bcc-94a7-fdb58fcf1f84 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: b802602d-f877-40b1-9a89-e26ec27b66ca kind: Event lastTimestamp: "2026-04-23T13:43:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:58Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6.18a900486859c89d namespace: kserve-ci-e2e-test resourceVersion: "15488" uid: b77d2b44-03f4-4da5-8ff4-f03596d68b84 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T13:44:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: b802602d-f877-40b1-9a89-e26ec27b66ca kind: Event lastTimestamp: "2026-04-23T13:45:36Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:44:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:36Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6.18a90048d77ed6a5 namespace: kserve-ci-e2e-test resourceVersion: "16388" uid: d75cae38-7dbe-49b3-a1b6-f1356833c207 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: b802602d-f877-40b1-9a89-e26ec27b66ca kind: Event lastTimestamp: "2026-04-23T13:45:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:34Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6.18a9005e9f749270 namespace: kserve-ci-e2e-test resourceVersion: "16351" uid: f9c471c3-f110-4ed6-9c05-6779291d1601 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: b802602d-f877-40b1-9a89-e26ec27b66ca kind: Event lastTimestamp: "2026-04-23T13:45:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:34Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6.18a9005e9f77be4a namespace: kserve-ci-e2e-test resourceVersion: "16352" uid: d094b826-2a1c-4ddc-b0ab-7d3b626a901b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: b802602d-f877-40b1-9a89-e26ec27b66ca kind: Event lastTimestamp: "2026-04-23T13:45:36Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:36Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6.18a9005f31b44a3d namespace: kserve-ci-e2e-test resourceVersion: "16386" uid: 07cf316a-478e-4048-830c-1794bc603267 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865 namespace: kserve-ci-e2e-test resourceVersion: "15349" uid: f4ec2f05-5f24-4639-95ed-19a698952019 kind: Event lastTimestamp: "2026-04-23T13:43:53Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865-b6wz6' metadata: creationTimestamp: "2026-04-23T13:43:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:53Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865.18a900473e8b5884 namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: c93f4fdb-ede9-427e-b36c-03e1d22fe98f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-4f8db-predictor namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: a60ee363-a960-477a-aefe-8912c6f5184a kind: Event lastTimestamp: "2026-04-23T13:43:53Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-4f8db-predictor-bdfb6b865 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:43:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:53Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor.18a900473a5250fe namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 9582d274-1875-4283-a9c9-116b2c6aad6f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:44:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-4f8db-predictor namespace: kserve-ci-e2e-test resourceVersion: "15347" uid: db739ee6-e606-4a95-8dc3-34947ce2e038 kind: Event lastTimestamp: "2026-04-23T13:44:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:44:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:38Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor.18a9004ab8a70f55 namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: 7ff87390-4fdd-41a9-a2da-b5a6db750b0c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:44:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-4f8db-predictor namespace: kserve-ci-e2e-test resourceVersion: "15347" uid: db739ee6-e606-4a95-8dc3-34947ce2e038 kind: Event lastTimestamp: "2026-04-23T13:44:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:44:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:44:38Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor.18a9004ab9043c3a namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: 2334368b-fa58-453a-88af-ca2c68aaf677 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:44:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-4f8db-predictor namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: db739ee6-e606-4a95-8dc3-34947ce2e038 kind: Event lastTimestamp: "2026-04-23T13:45:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:44:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:23Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor.18a9004e37b6eb05 namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: 93b74625-e5c8-4873-928c-adbfb74ce58e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:44:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-4f8db-predictor namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: db739ee6-e606-4a95-8dc3-34947ce2e038 kind: Event lastTimestamp: "2026-04-23T13:45:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:44:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:23Z" name: isvc-xgboost-graph-raw-hpa-4f8db-predictor.18a9004e38318a4d namespace: kserve-ci-e2e-test resourceVersion: "16177" uid: 8aabb561-64f7-406f-86e2-383e445c5beb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-23T13:43:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-4f8db namespace: kserve-ci-e2e-test resourceVersion: "15336" uid: abc2af42-9826-435f-88e8-032f93f8f14a kind: Event lastTimestamp: "2026-04-23T13:45:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:43:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:45:07Z" name: isvc-xgboost-graph-raw-hpa-4f8db.18a900473a018bc5 namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: 93398241-0102-47df-b4c9-941bdc497bfa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-4f8db namespace: kserve-ci-e2e-test resourceVersion: "16021" uid: abc2af42-9826-435f-88e8-032f93f8f14a kind: Event lastTimestamp: "2026-04-23T13:45:06Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-4f8db] is Ready metadata: creationTimestamp: "2026-04-23T13:45:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:45:06Z" name: isvc-xgboost-graph-raw-hpa-4f8db.18a900583d07d49e namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: 10252a5e-d694-46f5-a641-0f854022aa3c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:45:34.311429Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd namespace: kserve-ci-e2e-test resourceVersion: "16324" uid: 00dc171a-3be5-4310-91e3-38e0a4724108 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-35dde-predictor-5776cbf779-g8nmd to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-04-23T13:45:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:45:34Z" name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd.18a9005e9b939cab namespace: kserve-ci-e2e-test resourceVersion: "16333" uid: f644bca8-ed77-46fb-a141-3fc088d6e23c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-78b4f5d84d-ssxfd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:34Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd namespace: kserve-ci-e2e-test resourceVersion: "16344" uid: 00dc171a-3be5-4310-91e3-38e0a4724108 kind: Event lastTimestamp: "2026-04-23T13:45:34Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:45:34Z" name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd.18a9005eb547e650 namespace: kserve-ci-e2e-test resourceVersion: "16356" uid: 8c004519-28f5-46cb-8b51-9d873f67cd90 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: 00dc171a-3be5-4310-91e3-38e0a4724108 kind: Event lastTimestamp: "2026-04-23T13:45:34Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-23T13:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:34Z" name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd.18a9005eb6d2d721 namespace: kserve-ci-e2e-test resourceVersion: "16358" uid: 5c60fa6f-7562-46a0-b34e-cca905fdd708 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: 00dc171a-3be5-4310-91e3-38e0a4724108 kind: Event lastTimestamp: "2026-04-23T13:45:35Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.04s (1.04s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-23T13:45:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:35Z" name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd.18a9005ef4de9f70 namespace: kserve-ci-e2e-test resourceVersion: "16365" uid: 3e1f3e88-1998-40ec-9832-1a6e29f09407 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: 00dc171a-3be5-4310-91e3-38e0a4724108 kind: Event lastTimestamp: "2026-04-23T13:45:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:45:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:35Z" name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd.18a9005ef7f24be7 namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: 4bea9db9-7c9c-451f-86ed-322149026efc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: 00dc171a-3be5-4310-91e3-38e0a4724108 kind: Event lastTimestamp: "2026-04-23T13:45:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:45:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:35Z" name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd.18a9005ef8736ded namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: 7a595832-8db1-4ce2-826a-7b7f73ff6cb7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: 00dc171a-3be5-4310-91e3-38e0a4724108 kind: Event lastTimestamp: "2026-04-23T13:45:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:45:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:35Z" name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd.18a9005ef8840875 namespace: kserve-ci-e2e-test resourceVersion: "16370" uid: 8568c105-6672-4d9b-a845-4dd8fa5995d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: 00dc171a-3be5-4310-91e3-38e0a4724108 kind: Event lastTimestamp: "2026-04-23T13:45:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:45:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:35Z" name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd.18a9005efbf17632 namespace: kserve-ci-e2e-test resourceVersion: "16371" uid: 80bdb437-0eea-4a1c-bf0a-577b28cfc3a4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: 00dc171a-3be5-4310-91e3-38e0a4724108 kind: Event lastTimestamp: "2026-04-23T13:45:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:45:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:35Z" name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd.18a9005efc6f03cd namespace: kserve-ci-e2e-test resourceVersion: "16372" uid: f55b2008-9c2f-4c12-b6de-edf4268ab1ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:47:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: 00dc171a-3be5-4310-91e3-38e0a4724108 kind: Event lastTimestamp: "2026-04-23T13:47:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:47:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:47:09Z" name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd.18a90074c96b8b1a namespace: kserve-ci-e2e-test resourceVersion: "17215" uid: d25843b7-d847-473c-9fe7-14355e0f9f87 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:47:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: 00dc171a-3be5-4310-91e3-38e0a4724108 kind: Event lastTimestamp: "2026-04-23T13:47:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:47:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:47:09Z" name: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd.18a90074c96c38eb namespace: kserve-ci-e2e-test resourceVersion: "17218" uid: be797d03-303a-47c8-a691-3e5d56eb3519 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-35dde-predictor-5776cbf779 namespace: kserve-ci-e2e-test resourceVersion: "16304" uid: 5ac479f5-91bf-4372-9821-043d82fc0260 kind: Event lastTimestamp: "2026-04-23T13:45:34Z" message: 'Created pod: message-dumper-raw-35dde-predictor-5776cbf779-g8nmd' metadata: creationTimestamp: "2026-04-23T13:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:34Z" name: message-dumper-raw-35dde-predictor-5776cbf779.18a9005e9acd5fbd namespace: kserve-ci-e2e-test resourceVersion: "16326" uid: 5fd8e69d-0b64-4e5a-a388-ca7ce4360a52 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-35dde-predictor namespace: kserve-ci-e2e-test resourceVersion: "16296" uid: 00e01196-dd3a-476c-87db-77f43ddd31c6 kind: Event lastTimestamp: "2026-04-23T13:45:34Z" message: Scaled up replica set message-dumper-raw-35dde-predictor-5776cbf779 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:34Z" name: message-dumper-raw-35dde-predictor.18a9005e95aae395 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: 56ee41cb-6c7d-4f74-9df1-1dc49123020f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:45:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-35dde-predictor namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 8295bbf9-c00f-472f-a68f-b715beae8d1d kind: Event lastTimestamp: "2026-04-23T13:46:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:46:04Z" name: message-dumper-raw-35dde-predictor.18a90062144cd392 namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: 3bc8cc8c-f68d-4d59-959e-8545090fec2c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-23T13:45:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-35dde-predictor namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 8295bbf9-c00f-472f-a68f-b715beae8d1d kind: Event lastTimestamp: "2026-04-23T13:46:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:45:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:46:04Z" name: message-dumper-raw-35dde-predictor.18a9006214d2b67e namespace: kserve-ci-e2e-test resourceVersion: "16673" uid: fcc358cf-4a72-4788-a54b-8af5eeb28fc1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:45:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-35dde namespace: kserve-ci-e2e-test resourceVersion: "16283" uid: 4845ad8e-2f58-42f7-886c-ddab90ae88ba kind: Event lastTimestamp: "2026-04-23T13:45:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:45:43Z" name: message-dumper-raw-35dde.18a9005e9593f983 namespace: kserve-ci-e2e-test resourceVersion: "16445" uid: 2da27ed1-be42-48df-872d-b88914f69c99 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-35dde namespace: kserve-ci-e2e-test resourceVersion: "16283" uid: 4845ad8e-2f58-42f7-886c-ddab90ae88ba kind: Event lastTimestamp: "2026-04-23T13:45:34Z" message: 'Failed to update status for InferenceService "message-dumper-raw-35dde": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-35dde": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:45:34Z" name: message-dumper-raw-35dde.18a9005e9647af71 namespace: kserve-ci-e2e-test resourceVersion: "16313" uid: 07f52fbb-c107-4163-84b9-bb2d1f7945ae reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-35dde namespace: kserve-ci-e2e-test resourceVersion: "16443" uid: 4845ad8e-2f58-42f7-886c-ddab90ae88ba kind: Event lastTimestamp: "2026-04-23T13:45:43Z" message: InferenceService [message-dumper-raw-35dde] is Ready metadata: creationTimestamp: "2026-04-23T13:45:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:45:43Z" name: message-dumper-raw-35dde.18a90060b094053d namespace: kserve-ci-e2e-test resourceVersion: "16444" uid: 70cb6373-9bec-450c-9281-40d8110e8c13 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:43:43.567477Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-b90a6-98bfc68fb-jbrkv namespace: kserve-ci-e2e-test resourceVersion: "15178" uid: f7b430e5-15a1-4362-80da-08e5986dcdba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-b90a6-98bfc68fb-jbrkv to ip-10-0-134-22.ec2.internal metadata: creationTimestamp: "2026-04-23T13:43:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:43:43Z" name: model-chainer-raw-b90a6-98bfc68fb-jbrkv.18a90044d2b903aa namespace: kserve-ci-e2e-test resourceVersion: "15182" uid: 91ebf06b-8f51-4e1e-9a0c-3b6d4f1eb6f8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-78b4f5d84d-ssxfd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:43Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-b90a6-98bfc68fb-jbrkv namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: f7b430e5-15a1-4362-80da-08e5986dcdba kind: Event lastTimestamp: "2026-04-23T13:43:43Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:43:43Z" name: model-chainer-raw-b90a6-98bfc68fb-jbrkv.18a90044eb98df25 namespace: kserve-ci-e2e-test resourceVersion: "15197" uid: 8edfc12e-3f50-4d77-95c1-dcb2d7552d65 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-b90a6} kind: Pod name: model-chainer-raw-b90a6-98bfc68fb-jbrkv namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: f7b430e5-15a1-4362-80da-08e5986dcdba kind: Event lastTimestamp: "2026-04-23T13:43:44Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:64310d078bf386870e904a5a4ca9cf4559d7cad81ce46ab7ca9a6afadbba1f7c" metadata: creationTimestamp: "2026-04-23T13:43:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:44Z" name: model-chainer-raw-b90a6-98bfc68fb-jbrkv.18a90044ed0baa92 namespace: kserve-ci-e2e-test resourceVersion: "15199" uid: 3d7e3966-c349-4501-964a-238ac5cd9efb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-22.ec2.internal source: component: kubelet host: ip-10-0-134-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-b90a6} kind: Pod name: model-chainer-raw-b90a6-98bfc68fb-jbrkv namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: f7b430e5-15a1-4362-80da-08e5986dcdba kind: Event lastTimestamp: "2026-04-23T13:43:46Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:64310d078bf386870e904a5a4ca9cf4559d7cad81ce46ab7ca9a6afadbba1f7c" in 2.238s (2.238s including waiting). Image size: 216346181 bytes.' metadata: creationTimestamp: "2026-04-23T13:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:46Z" name: model-chainer-raw-b90a6-98bfc68fb-jbrkv.18a9004572787cb1 namespace: kserve-ci-e2e-test resourceVersion: "15214" uid: 32cf1fae-258a-47cc-80ae-6920c93bd932 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-22.ec2.internal source: component: kubelet host: ip-10-0-134-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-b90a6} kind: Pod name: model-chainer-raw-b90a6-98bfc68fb-jbrkv namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: f7b430e5-15a1-4362-80da-08e5986dcdba kind: Event lastTimestamp: "2026-04-23T13:43:46Z" message: 'Created container: model-chainer-raw-b90a6' metadata: creationTimestamp: "2026-04-23T13:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:46Z" name: model-chainer-raw-b90a6-98bfc68fb-jbrkv.18a90045766d88b8 namespace: kserve-ci-e2e-test resourceVersion: "15216" uid: 66e023d1-8c84-4f7f-ba68-572259dbf12a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-22.ec2.internal source: component: kubelet host: ip-10-0-134-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-b90a6} kind: Pod name: model-chainer-raw-b90a6-98bfc68fb-jbrkv namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: f7b430e5-15a1-4362-80da-08e5986dcdba kind: Event lastTimestamp: "2026-04-23T13:43:46Z" message: Started container model-chainer-raw-b90a6 metadata: creationTimestamp: "2026-04-23T13:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:46Z" name: model-chainer-raw-b90a6-98bfc68fb-jbrkv.18a9004576e1a3e5 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 50855950-bc7e-4ad8-a293-2033ffab3eb5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-22.ec2.internal source: component: kubelet host: ip-10-0-134-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-b90a6} kind: Pod name: model-chainer-raw-b90a6-98bfc68fb-jbrkv namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: f7b430e5-15a1-4362-80da-08e5986dcdba kind: Event lastTimestamp: "2026-04-23T13:43:53Z" message: Stopping container model-chainer-raw-b90a6 metadata: creationTimestamp: "2026-04-23T13:43:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:43:53Z" name: model-chainer-raw-b90a6-98bfc68fb-jbrkv.18a900472a570758 namespace: kserve-ci-e2e-test resourceVersion: "15293" uid: 3cf1bdd8-7461-4e74-8674-ac34830251a2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-22.ec2.internal source: component: kubelet host: ip-10-0-134-22.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:43:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-b90a6} kind: Pod name: model-chainer-raw-b90a6-98bfc68fb-jbrkv namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: f7b430e5-15a1-4362-80da-08e5986dcdba kind: Event lastTimestamp: "2026-04-23T13:44:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T13:43:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:44:23Z" name: model-chainer-raw-b90a6-98bfc68fb-jbrkv.18a9004841b8d7c9 namespace: kserve-ci-e2e-test resourceVersion: "15680" uid: cc907638-3c63-46c4-b257-ea3d3bfdce3a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-22.ec2.internal source: component: kubelet host: ip-10-0-134-22.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-b90a6-98bfc68fb namespace: kserve-ci-e2e-test resourceVersion: "15174" uid: 5c08cf12-be89-4d9d-b5c5-0ca57903e5af kind: Event lastTimestamp: "2026-04-23T13:43:43Z" message: 'Created pod: model-chainer-raw-b90a6-98bfc68fb-jbrkv' metadata: creationTimestamp: "2026-04-23T13:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:43Z" name: model-chainer-raw-b90a6-98bfc68fb.18a90044d22f95ed namespace: kserve-ci-e2e-test resourceVersion: "15179" uid: f83c5f31-f8d4-45f6-9c60-03027e49463c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-b90a6 namespace: kserve-ci-e2e-test resourceVersion: "15172" uid: 77fb35f6-8845-44d9-b220-184aa7bc93cf kind: Event lastTimestamp: "2026-04-23T13:43:43Z" message: Scaled up replica set model-chainer-raw-b90a6-98bfc68fb from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:43:43Z" name: model-chainer-raw-b90a6.18a90044d10cf6b5 namespace: kserve-ci-e2e-test resourceVersion: "15175" uid: bdbdf10e-3520-4490-8279-3ad6867cdc54 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:43:43Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-b90a6 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: a1596d52-fbdc-41c9-a3a0-161c3ca37cf4 kind: Event lastTimestamp: "2026-04-23T13:43:43Z" message: InferenceGraph [model-chainer-raw-b90a6] is Ready metadata: creationTimestamp: "2026-04-23T13:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:43:43Z" name: model-chainer-raw-b90a6.18a90044d67e708e namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: f95890fd-cea7-4f29-94f1-5a33de43c312 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:45:23.930055Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-4f8db-7f97674dc6-bphnl namespace: kserve-ci-e2e-test resourceVersion: "16162" uid: cdffad98-406e-4b80-ab90-90779209ae8c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-4f8db-7f97674dc6-bphnl to ip-10-0-134-22.ec2.internal metadata: creationTimestamp: "2026-04-23T13:45:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:45:23Z" name: model-chainer-raw-hpa-4f8db-7f97674dc6-bphnl.18a9005c30cc65e8 namespace: kserve-ci-e2e-test resourceVersion: "16167" uid: 09770297-64f4-4278-9f76-f1056bbb87b5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-78b4f5d84d-ssxfd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:24Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-4f8db-7f97674dc6-bphnl namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: cdffad98-406e-4b80-ab90-90779209ae8c kind: Event lastTimestamp: "2026-04-23T13:45:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-4f8db-serving-cert" not found' metadata: creationTimestamp: "2026-04-23T13:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:24Z" name: model-chainer-raw-hpa-4f8db-7f97674dc6-bphnl.18a9005c3b85bfbe namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: e3489425-90ec-4ab6-aa48-34b5ce62bf89 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-22.ec2.internal source: component: kubelet host: ip-10-0-134-22.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:24Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-4f8db-7f97674dc6-bphnl namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: cdffad98-406e-4b80-ab90-90779209ae8c kind: Event lastTimestamp: "2026-04-23T13:45:24Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:45:24Z" name: model-chainer-raw-hpa-4f8db-7f97674dc6-bphnl.18a9005c6d7b18ee namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: 94254d89-41ac-45c0-b692-a9abe6685b7b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-4f8db} kind: Pod name: model-chainer-raw-hpa-4f8db-7f97674dc6-bphnl namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: cdffad98-406e-4b80-ab90-90779209ae8c kind: Event lastTimestamp: "2026-04-23T13:45:24Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:64310d078bf386870e904a5a4ca9cf4559d7cad81ce46ab7ca9a6afadbba1f7c" already present on machine metadata: creationTimestamp: "2026-04-23T13:45:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:24Z" name: model-chainer-raw-hpa-4f8db-7f97674dc6-bphnl.18a9005c6ee11757 namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: 02b085f0-416c-4280-8d52-382aa2c0fa49 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-22.ec2.internal source: component: kubelet host: ip-10-0-134-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-4f8db} kind: Pod name: model-chainer-raw-hpa-4f8db-7f97674dc6-bphnl namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: cdffad98-406e-4b80-ab90-90779209ae8c kind: Event lastTimestamp: "2026-04-23T13:45:25Z" message: 'Created container: model-chainer-raw-hpa-4f8db' metadata: creationTimestamp: "2026-04-23T13:45:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:25Z" name: model-chainer-raw-hpa-4f8db-7f97674dc6-bphnl.18a9005c72d28806 namespace: kserve-ci-e2e-test resourceVersion: "16193" uid: 189ae998-dac9-4c34-b886-1c366b58c4f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-22.ec2.internal source: component: kubelet host: ip-10-0-134-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-4f8db} kind: Pod name: model-chainer-raw-hpa-4f8db-7f97674dc6-bphnl namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: cdffad98-406e-4b80-ab90-90779209ae8c kind: Event lastTimestamp: "2026-04-23T13:45:25Z" message: Started container model-chainer-raw-hpa-4f8db metadata: creationTimestamp: "2026-04-23T13:45:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:25Z" name: model-chainer-raw-hpa-4f8db-7f97674dc6-bphnl.18a9005c735702af namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 8914d68e-b98a-453c-882e-29cbcf9c3ec9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-22.ec2.internal source: component: kubelet host: ip-10-0-134-22.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-4f8db} kind: Pod name: model-chainer-raw-hpa-4f8db-7f97674dc6-bphnl namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: cdffad98-406e-4b80-ab90-90779209ae8c kind: Event lastTimestamp: "2026-04-23T13:45:34Z" message: Stopping container model-chainer-raw-hpa-4f8db metadata: creationTimestamp: "2026-04-23T13:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:45:34Z" name: model-chainer-raw-hpa-4f8db-7f97674dc6-bphnl.18a9005e8bd7c12f namespace: kserve-ci-e2e-test resourceVersion: "16280" uid: 182294a7-73ff-49a8-8619-20be8d82c1a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-22.ec2.internal source: component: kubelet host: ip-10-0-134-22.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-23T13:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-4f8db} kind: Pod name: model-chainer-raw-hpa-4f8db-7f97674dc6-bphnl namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: cdffad98-406e-4b80-ab90-90779209ae8c kind: Event lastTimestamp: "2026-04-23T13:46:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-23T13:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:46:01Z" name: model-chainer-raw-hpa-4f8db-7f97674dc6-bphnl.18a9005f2588029f namespace: kserve-ci-e2e-test resourceVersion: "16653" uid: e454d6e4-3ee3-4003-b935-d494c66eb9b7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-22.ec2.internal source: component: kubelet host: ip-10-0-134-22.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-4f8db-7f97674dc6 namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 6c4a054b-0f97-4e78-98d0-1a76ef8b3f0c kind: Event lastTimestamp: "2026-04-23T13:45:23Z" message: 'Created pod: model-chainer-raw-hpa-4f8db-7f97674dc6-bphnl' metadata: creationTimestamp: "2026-04-23T13:45:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:23Z" name: model-chainer-raw-hpa-4f8db-7f97674dc6.18a9005c30581868 namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: 0f01526b-3be8-4e3c-8cfa-28b46783d339 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-4f8db namespace: kserve-ci-e2e-test resourceVersion: "16155" uid: cad853ef-7ce8-4df7-952d-dd79b2826bb3 kind: Event lastTimestamp: "2026-04-23T13:45:23Z" message: Scaled up replica set model-chainer-raw-hpa-4f8db-7f97674dc6 from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:45:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:45:23Z" name: model-chainer-raw-hpa-4f8db.18a9005c2f39e3c9 namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 2f50d006-43be-4238-88df-8c138fb0a441 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:45:23Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-4f8db namespace: kserve-ci-e2e-test resourceVersion: "16173" uid: bdff3549-242f-40c0-90e4-61351d54eda0 kind: Event lastTimestamp: "2026-04-23T13:45:23Z" message: InferenceGraph [model-chainer-raw-hpa-4f8db] is Ready metadata: creationTimestamp: "2026-04-23T13:45:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:45:23Z" name: model-chainer-raw-hpa-4f8db.18a9005c338942dc namespace: kserve-ci-e2e-test resourceVersion: "16175" uid: 99a5cabf-b096-48a8-9875-dfb0c23e34da reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:51:36.984563Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: f6142f85-8f3b-40f6-92d3-9758f42d36ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-ed1e5-predictor-76464db46d-qpzqt to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-04-23T13:51:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:51:36Z" name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt.18a900b30c946359 namespace: kserve-ci-e2e-test resourceVersion: "19471" uid: 15c4c447-3fe7-461e-b657-c1512c57279e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-78b4f5d84d-ssxfd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:37Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt namespace: kserve-ci-e2e-test resourceVersion: "19474" uid: f6142f85-8f3b-40f6-92d3-9758f42d36ba kind: Event lastTimestamp: "2026-04-23T13:51:37Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:51:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:51:37Z" name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt.18a900b325de4965 namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: 526a3ead-fb6f-48f1-bf3c-a3ffbc165687 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: f6142f85-8f3b-40f6-92d3-9758f42d36ba kind: Event lastTimestamp: "2026-04-23T13:51:37Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bfc8f044ee2a84e90b7b9876040a412b44d7b121a59a5fe7b22a946b15b61519" already present on machine metadata: creationTimestamp: "2026-04-23T13:51:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:37Z" name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt.18a900b3271ab666 namespace: kserve-ci-e2e-test resourceVersion: "19483" uid: 3821cd58-d5f3-4d98-9d27-d8852f8599f3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: f6142f85-8f3b-40f6-92d3-9758f42d36ba kind: Event lastTimestamp: "2026-04-23T13:51:37Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T13:51:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:37Z" name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt.18a900b32ab6b5be namespace: kserve-ci-e2e-test resourceVersion: "19486" uid: 788dfd12-2148-478c-bfba-cfb7f982a111 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:37Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: f6142f85-8f3b-40f6-92d3-9758f42d36ba kind: Event lastTimestamp: "2026-04-23T13:51:37Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T13:51:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:37Z" name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt.18a900b32b2c8394 namespace: kserve-ci-e2e-test resourceVersion: "19487" uid: 7f9864ea-f15c-45b3-a6ce-0c825e3855af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: f6142f85-8f3b-40f6-92d3-9758f42d36ba kind: Event lastTimestamp: "2026-04-23T13:51:41Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T13:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:41Z" name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt.18a900b41ad46bcd namespace: kserve-ci-e2e-test resourceVersion: "19526" uid: 8435f1bd-9314-49d5-996e-1386a54eccd1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: f6142f85-8f3b-40f6-92d3-9758f42d36ba kind: Event lastTimestamp: "2026-04-23T13:51:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:41Z" name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt.18a900b41df532ea namespace: kserve-ci-e2e-test resourceVersion: "19530" uid: 02796a8f-f82b-4882-9afc-5a5909e1ce40 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: f6142f85-8f3b-40f6-92d3-9758f42d36ba kind: Event lastTimestamp: "2026-04-23T13:51:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:41Z" name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt.18a900b41e758034 namespace: kserve-ci-e2e-test resourceVersion: "19531" uid: fecee35c-73ef-45e9-8d00-32d509750dc5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: f6142f85-8f3b-40f6-92d3-9758f42d36ba kind: Event lastTimestamp: "2026-04-23T13:51:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:41Z" name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt.18a900b41e83e2bb namespace: kserve-ci-e2e-test resourceVersion: "19532" uid: c4c9766c-0ed4-4375-aac8-03e885146813 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: f6142f85-8f3b-40f6-92d3-9758f42d36ba kind: Event lastTimestamp: "2026-04-23T13:51:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:41Z" name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt.18a900b421b5e313 namespace: kserve-ci-e2e-test resourceVersion: "19534" uid: 25978124-7f75-4f8c-8b46-3f3dd8ff9d85 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: f6142f85-8f3b-40f6-92d3-9758f42d36ba kind: Event lastTimestamp: "2026-04-23T13:51:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:51:41Z" name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt.18a900b422439111 namespace: kserve-ci-e2e-test resourceVersion: "19535" uid: 32733235-70fc-4663-8ab0-24ae9b836121 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T13:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: f6142f85-8f3b-40f6-92d3-9758f42d36ba kind: Event lastTimestamp: "2026-04-23T13:52:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:58Z" name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt.18a900b456da85fc namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: 3d424d06-16b4-4b7a-9d87-32005f5547b3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: f6142f85-8f3b-40f6-92d3-9758f42d36ba kind: Event lastTimestamp: "2026-04-23T13:52:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:56Z" name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt.18a900c58df873de namespace: kserve-ci-e2e-test resourceVersion: "20105" uid: 2459bab2-b7ce-474e-a304-e510213730d1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: f6142f85-8f3b-40f6-92d3-9758f42d36ba kind: Event lastTimestamp: "2026-04-23T13:52:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:56Z" name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt.18a900c58dfb1797 namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 552cabab-7154-460c-846a-c007b0d348b8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt namespace: kserve-ci-e2e-test resourceVersion: "19468" uid: f6142f85-8f3b-40f6-92d3-9758f42d36ba kind: Event lastTimestamp: "2026-04-23T13:52:58Z" message: 'Readiness probe failed: Get "https://10.134.0.44:8643/healthz": dial tcp 10.134.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:52:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:58Z" name: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt.18a900c608f5e9dd namespace: kserve-ci-e2e-test resourceVersion: "20158" uid: 14331af8-dad1-4487-9c5a-f22d482321fb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-ed1e5-predictor-76464db46d namespace: kserve-ci-e2e-test resourceVersion: "19450" uid: 400916fb-8f42-43e8-aec7-0f7800878a9a kind: Event lastTimestamp: "2026-04-23T13:51:36Z" message: 'Created pod: raw-sklearn-ed1e5-predictor-76464db46d-qpzqt' metadata: creationTimestamp: "2026-04-23T13:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:51:36Z" name: raw-sklearn-ed1e5-predictor-76464db46d.18a900b30bfe1ad7 namespace: kserve-ci-e2e-test resourceVersion: "19467" uid: ed674c7b-121e-49f7-b4d6-da3e75f99e7c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-ed1e5-predictor namespace: kserve-ci-e2e-test resourceVersion: "19449" uid: 33b517a2-e04c-410a-9238-50f41a15357a kind: Event lastTimestamp: "2026-04-23T13:51:36Z" message: Scaled up replica set raw-sklearn-ed1e5-predictor-76464db46d from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:51:36Z" name: raw-sklearn-ed1e5-predictor.18a900b2f2c2ab32 namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: 8dd06f2b-fe94-4095-ac7b-f72a3fb18ab4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-ed1e5-predictor namespace: kserve-ci-e2e-test resourceVersion: "19451" uid: ac7bbe4d-47ff-4db4-af4a-c1b04eb9ec35 kind: Event lastTimestamp: "2026-04-23T13:51:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:51:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:51:51Z" name: raw-sklearn-ed1e5-predictor.18a900b671afcca6 namespace: kserve-ci-e2e-test resourceVersion: "19613" uid: 1f7c09eb-c774-43f8-89f1-1b41e91658e0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-ed1e5-predictor namespace: kserve-ci-e2e-test resourceVersion: "19451" uid: ac7bbe4d-47ff-4db4-af4a-c1b04eb9ec35 kind: Event lastTimestamp: "2026-04-23T13:51:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:51:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:51:51Z" name: raw-sklearn-ed1e5-predictor.18a900b672145c94 namespace: kserve-ci-e2e-test resourceVersion: "19615" uid: 84e9aa70-d028-4ee0-9d82-f98b74480e64 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:52:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-ed1e5-predictor namespace: kserve-ci-e2e-test resourceVersion: "19614" uid: ac7bbe4d-47ff-4db4-af4a-c1b04eb9ec35 kind: Event lastTimestamp: "2026-04-23T13:52:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:52:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:52:51Z" name: raw-sklearn-ed1e5-predictor.18a900b9f1233635 namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 8d0b0767-5f14-4caf-b644-48309ceb5202 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:52:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-ed1e5-predictor namespace: kserve-ci-e2e-test resourceVersion: "19614" uid: ac7bbe4d-47ff-4db4-af4a-c1b04eb9ec35 kind: Event lastTimestamp: "2026-04-23T13:52:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:52:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:52:51Z" name: raw-sklearn-ed1e5-predictor.18a900b9f18d1bb0 namespace: kserve-ci-e2e-test resourceVersion: "20043" uid: 6ae151f9-d102-4a92-bed6-21b96f155123 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-23T13:51:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-ed1e5 namespace: kserve-ci-e2e-test resourceVersion: "19436" uid: 863281bd-bf38-4e4c-a9fd-11b3a9f9b41e kind: Event lastTimestamp: "2026-04-23T13:52:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:52:48Z" name: raw-sklearn-ed1e5.18a900b2f2fe271c namespace: kserve-ci-e2e-test resourceVersion: "20021" uid: db48962e-d05f-42cd-ad28-4ebcb53b0753 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:51:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-ed1e5 namespace: kserve-ci-e2e-test resourceVersion: "19436" uid: 863281bd-bf38-4e4c-a9fd-11b3a9f9b41e kind: Event lastTimestamp: "2026-04-23T13:51:36Z" message: 'Failed to update status for InferenceService "raw-sklearn-ed1e5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-ed1e5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:51:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:51:36Z" name: raw-sklearn-ed1e5.18a900b2f35eee62 namespace: kserve-ci-e2e-test resourceVersion: "19462" uid: 9ec25620-f43c-4656-b022-8c637eaf2313 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-ed1e5 namespace: kserve-ci-e2e-test resourceVersion: "20019" uid: 863281bd-bf38-4e4c-a9fd-11b3a9f9b41e kind: Event lastTimestamp: "2026-04-23T13:52:48Z" message: InferenceService [raw-sklearn-ed1e5] is Ready metadata: creationTimestamp: "2026-04-23T13:52:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:52:48Z" name: raw-sklearn-ed1e5.18a900c3b9fcdc80 namespace: kserve-ci-e2e-test resourceVersion: "20020" uid: ec70f191-49eb-4f04-beb1-a86aa9d6538b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-23T13:52:56.590507Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2 namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: 70a1e324-9b25-4833-8a42-24a835018067 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2 to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-04-23T13:52:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-23T13:52:56Z" name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2.18a900c59576c3c9 namespace: kserve-ci-e2e-test resourceVersion: "20126" uid: e71cefa3-fe96-492a-96ab-19a117516564 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-78b4f5d84d-ssxfd source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:57Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2 namespace: kserve-ci-e2e-test resourceVersion: "20129" uid: 70a1e324-9b25-4833-8a42-24a835018067 kind: Event lastTimestamp: "2026-04-23T13:52:57Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-23T13:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-23T13:52:57Z" name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2.18a900c5afbd8b94 namespace: kserve-ci-e2e-test resourceVersion: "20137" uid: 30982a53-4b77-4220-99a1-d707c33d3beb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 70a1e324-9b25-4833-8a42-24a835018067 kind: Event lastTimestamp: "2026-04-23T13:52:57Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:bfc8f044ee2a84e90b7b9876040a412b44d7b121a59a5fe7b22a946b15b61519" already present on machine metadata: creationTimestamp: "2026-04-23T13:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:57Z" name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2.18a900c5b10f2193 namespace: kserve-ci-e2e-test resourceVersion: "20139" uid: d6c172ea-d638-451b-acee-6d3d6d7a63ea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 70a1e324-9b25-4833-8a42-24a835018067 kind: Event lastTimestamp: "2026-04-23T13:52:57Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-23T13:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:57Z" name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2.18a900c5b3dd41a3 namespace: kserve-ci-e2e-test resourceVersion: "20141" uid: acf6cdb9-4eca-4f6b-b6e9-755ee58bbc9e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 70a1e324-9b25-4833-8a42-24a835018067 kind: Event lastTimestamp: "2026-04-23T13:52:57Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-23T13:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:52:57Z" name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2.18a900c5b4610df0 namespace: kserve-ci-e2e-test resourceVersion: "20142" uid: 1769b170-e210-4475-b393-e17b0bad8685 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 70a1e324-9b25-4833-8a42-24a835018067 kind: Event lastTimestamp: "2026-04-23T13:53:00Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1436" already present on machine metadata: creationTimestamp: "2026-04-23T13:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:00Z" name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2.18a900c69016b0be namespace: kserve-ci-e2e-test resourceVersion: "20177" uid: a3166d37-8ec7-4550-b32c-89dc9215749f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 70a1e324-9b25-4833-8a42-24a835018067 kind: Event lastTimestamp: "2026-04-23T13:53:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-23T13:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:00Z" name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2.18a900c693681193 namespace: kserve-ci-e2e-test resourceVersion: "20179" uid: 8c03cac2-d1f4-4904-8532-1a622ec88a34 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 70a1e324-9b25-4833-8a42-24a835018067 kind: Event lastTimestamp: "2026-04-23T13:53:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-23T13:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:00Z" name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2.18a900c693e78929 namespace: kserve-ci-e2e-test resourceVersion: "20182" uid: bc8c4dd4-59df-4202-843f-220f353533ed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 70a1e324-9b25-4833-8a42-24a835018067 kind: Event lastTimestamp: "2026-04-23T13:53:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-23T13:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:00Z" name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2.18a900c693f427fb namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: db658ee7-0d01-4974-910d-7bf3c6dae6c0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 70a1e324-9b25-4833-8a42-24a835018067 kind: Event lastTimestamp: "2026-04-23T13:53:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-23T13:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:00Z" name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2.18a900c697788406 namespace: kserve-ci-e2e-test resourceVersion: "20186" uid: 59888286-c0b4-4f9f-ba51-229ddd24efd7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 70a1e324-9b25-4833-8a42-24a835018067 kind: Event lastTimestamp: "2026-04-23T13:53:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:53:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:53:00Z" name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2.18a900c69806b21a namespace: kserve-ci-e2e-test resourceVersion: "20187" uid: 5ffc598c-256b-4ac5-806e-810b773f1704 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-23T13:53:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 70a1e324-9b25-4833-8a42-24a835018067 kind: Event lastTimestamp: "2026-04-23T13:54:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:53:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:18Z" name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2.18a900c7081037bb namespace: kserve-ci-e2e-test resourceVersion: "20806" uid: 23fbaa6d-a0ef-4654-be32-86682b75094b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 70a1e324-9b25-4833-8a42-24a835018067 kind: Event lastTimestamp: "2026-04-23T13:54:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-23T13:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:16Z" name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2.18a900d838addad7 namespace: kserve-ci-e2e-test resourceVersion: "20756" uid: 0900037a-9402-49e1-a611-1a0e67308a6c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 70a1e324-9b25-4833-8a42-24a835018067 kind: Event lastTimestamp: "2026-04-23T13:54:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-23T13:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:16Z" name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2.18a900d838ae7468 namespace: kserve-ci-e2e-test resourceVersion: "20757" uid: dbcd8066-2874-40ec-b91e-762529d43341 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 70a1e324-9b25-4833-8a42-24a835018067 kind: Event lastTimestamp: "2026-04-23T13:54:18Z" message: 'Readiness probe failed: Get "https://10.134.0.45:8643/healthz": dial tcp 10.134.0.45:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-23T13:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-23T13:54:18Z" name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2.18a900d8ba21377f namespace: kserve-ci-e2e-test resourceVersion: "20805" uid: 1a4e2c3d-6f19-4f0e-b7af-7fe5280b70de reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb namespace: kserve-ci-e2e-test resourceVersion: "20109" uid: eb36fd84-203b-4d45-9e06-31e5c0fa21a8 kind: Event lastTimestamp: "2026-04-23T13:52:56Z" message: 'Created pod: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb-n4hl2' metadata: creationTimestamp: "2026-04-23T13:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:52:56Z" name: raw-sklearn-runtime-b69cc-predictor-6c984fd4fb.18a900c5949906a6 namespace: kserve-ci-e2e-test resourceVersion: "20121" uid: 52aa0501-fcb9-4e8b-b4ca-51476c805a81 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-b69cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20106" uid: 076ecb9e-4a67-497f-8ddf-598c771eb7d2 kind: Event lastTimestamp: "2026-04-23T13:52:56Z" message: Scaled up replica set raw-sklearn-runtime-b69cc-predictor-6c984fd4fb from 0 to 1 metadata: creationTimestamp: "2026-04-23T13:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:52:56Z" name: raw-sklearn-runtime-b69cc-predictor.18a900c58f1462f5 namespace: kserve-ci-e2e-test resourceVersion: "20111" uid: 2546c0d7-bb67-4cae-b3ae-9ce00b6c265a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-b69cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20108" uid: d3dfded6-d2c7-477c-83ee-61e6c46e197f kind: Event lastTimestamp: "2026-04-23T13:53:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:11Z" name: raw-sklearn-runtime-b69cc-predictor.18a900c90d822ee6 namespace: kserve-ci-e2e-test resourceVersion: "20272" uid: a3d07af0-120f-4862-b7c8-3eb1389e1851 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:53:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-b69cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20108" uid: d3dfded6-d2c7-477c-83ee-61e6c46e197f kind: Event lastTimestamp: "2026-04-23T13:53:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-23T13:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:53:11Z" name: raw-sklearn-runtime-b69cc-predictor.18a900c90ddcdc04 namespace: kserve-ci-e2e-test resourceVersion: "20274" uid: 4d4a70c3-4139-4aff-b9ed-0c50fd4987b0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:53:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-b69cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20273" uid: d3dfded6-d2c7-477c-83ee-61e6c46e197f kind: Event lastTimestamp: "2026-04-23T13:54:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:53:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:11Z" name: raw-sklearn-runtime-b69cc-predictor.18a900cc8cfaccc9 namespace: kserve-ci-e2e-test resourceVersion: "20699" uid: d5e42ac7-7df7-4f6c-b7a4-c26a0e0a0b16 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-23T13:53:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-b69cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "20273" uid: d3dfded6-d2c7-477c-83ee-61e6c46e197f kind: Event lastTimestamp: "2026-04-23T13:54:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-23T13:53:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-23T13:54:11Z" name: raw-sklearn-runtime-b69cc-predictor.18a900cc8d6daf1d namespace: kserve-ci-e2e-test resourceVersion: "20700" uid: bf63f09f-6cea-47d6-87bf-7b1d7e75e7ed reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-23T13:52:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-b69cc namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: a7a1b939-3f6a-4e13-9527-b5fdf5fd5277 kind: Event lastTimestamp: "2026-04-23T13:54:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-23T13:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:54:08Z" name: raw-sklearn-runtime-b69cc.18a900c58eee6478 namespace: kserve-ci-e2e-test resourceVersion: "20681" uid: 94af533e-5a5f-4214-a3ec-7e4eea263678 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:52:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-b69cc namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: a7a1b939-3f6a-4e13-9527-b5fdf5fd5277 kind: Event lastTimestamp: "2026-04-23T13:52:56Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-b69cc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-b69cc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-23T13:52:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:52:56Z" name: raw-sklearn-runtime-b69cc.18a900c58f67bf25 namespace: kserve-ci-e2e-test resourceVersion: "20113" uid: 55dfca4a-44d0-417c-8ea3-a0cef73da305 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-23T13:54:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-b69cc namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: a7a1b939-3f6a-4e13-9527-b5fdf5fd5277 kind: Event lastTimestamp: "2026-04-23T13:54:08Z" message: InferenceService [raw-sklearn-runtime-b69cc] is Ready metadata: creationTimestamp: "2026-04-23T13:54:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-23T13:54:08Z" name: raw-sklearn-runtime-b69cc.18a900d66bb05036 namespace: kserve-ci-e2e-test resourceVersion: "20679" uid: e79040ad-7349-4098-ab9c-a240f51c25aa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "20954"