--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-06-02T16:05:14.070768Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-0b4fc4-predictor-9466c49c6-mt7xv namespace: kserve-ci-e2e-test resourceVersion: "18942" uid: 247c8ace-4646-4e2b-b9fb-f5afb563080c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-0b4fc4-predictor-9466c49c6-mt7xv to ip-10-0-140-246.ec2.internal metadata: creationTimestamp: "2026-06-02T16:05:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-02T16:05:14Z" name: isvc-init-fail-0b4fc4-predictor-9466c49c6-mt7xv.18b54f34607fbf81 namespace: kserve-ci-e2e-test resourceVersion: "18948" uid: 6f6c0765-f506-4663-a95e-d8633120df99 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8ddcd7f6c-hs6d2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:14Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-0b4fc4-predictor-9466c49c6-mt7xv namespace: kserve-ci-e2e-test resourceVersion: "18946" uid: 247c8ace-4646-4e2b-b9fb-f5afb563080c kind: Event lastTimestamp: "2026-06-02T16:05:14Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-init-fail-0b4fc4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-02T16:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:14Z" name: isvc-init-fail-0b4fc4-predictor-9466c49c6-mt7xv.18b54f346cefbb6f namespace: kserve-ci-e2e-test resourceVersion: "18956" uid: e679987b-720c-4506-b438-b2086f4df0be reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:15Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-0b4fc4-predictor-9466c49c6-mt7xv namespace: kserve-ci-e2e-test resourceVersion: "18952" uid: 247c8ace-4646-4e2b-b9fb-f5afb563080c kind: Event lastTimestamp: "2026-06-02T16:05:15Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-02T16:05:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-02T16:05:15Z" name: isvc-init-fail-0b4fc4-predictor-9466c49c6-mt7xv.18b54f349ed30326 namespace: kserve-ci-e2e-test resourceVersion: "18966" uid: 006004b8-2afd-4340-9660-7c61c1921416 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T16:05:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-0b4fc4-predictor-9466c49c6-mt7xv namespace: kserve-ci-e2e-test resourceVersion: "18946" uid: 247c8ace-4646-4e2b-b9fb-f5afb563080c kind: Event lastTimestamp: "2026-06-02T16:05:20Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ca9e7e1d07975dffacc4b293d655f2a02df4615629916a1cdc022036c5a27a57" already present on machine metadata: creationTimestamp: "2026-06-02T16:05:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:20Z" name: isvc-init-fail-0b4fc4-predictor-9466c49c6-mt7xv.18b54f34a06e3efd namespace: kserve-ci-e2e-test resourceVersion: "19021" uid: 13b88f98-2813-4ad8-b2b8-e799c1697901 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T16:05:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-0b4fc4-predictor-9466c49c6-mt7xv namespace: kserve-ci-e2e-test resourceVersion: "18946" uid: 247c8ace-4646-4e2b-b9fb-f5afb563080c kind: Event lastTimestamp: "2026-06-02T16:05:20Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-02T16:05:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:20Z" name: isvc-init-fail-0b4fc4-predictor-9466c49c6-mt7xv.18b54f34a41d0345 namespace: kserve-ci-e2e-test resourceVersion: "19023" uid: f66a3e36-c7fd-42d3-90ef-55648762b6b3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T16:05:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-0b4fc4-predictor-9466c49c6-mt7xv namespace: kserve-ci-e2e-test resourceVersion: "18946" uid: 247c8ace-4646-4e2b-b9fb-f5afb563080c kind: Event lastTimestamp: "2026-06-02T16:05:20Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-02T16:05:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:20Z" name: isvc-init-fail-0b4fc4-predictor-9466c49c6-mt7xv.18b54f34a4e3ca32 namespace: kserve-ci-e2e-test resourceVersion: "19024" uid: f8daccf6-bf4c-4c67-80ea-c0a08d79b25b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-0b4fc4-predictor-9466c49c6-mt7xv namespace: kserve-ci-e2e-test resourceVersion: "18946" uid: 247c8ace-4646-4e2b-b9fb-f5afb563080c kind: Event lastTimestamp: "2026-06-02T16:05:24Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-06-02T16:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:24Z" name: isvc-init-fail-0b4fc4-predictor-9466c49c6-mt7xv.18b54f36b3c3cecc namespace: kserve-ci-e2e-test resourceVersion: "19071" uid: 53180e01-d0a6-46f8-b002-437efbb7a07c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-0b4fc4-predictor-9466c49c6 namespace: kserve-ci-e2e-test resourceVersion: "18934" uid: 29599c13-7710-40ba-9a85-60f5fd0bb060 kind: Event lastTimestamp: "2026-06-02T16:05:14Z" message: 'Created pod: isvc-init-fail-0b4fc4-predictor-9466c49c6-mt7xv' metadata: creationTimestamp: "2026-06-02T16:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:05:14Z" name: isvc-init-fail-0b4fc4-predictor-9466c49c6.18b54f3460119210 namespace: kserve-ci-e2e-test resourceVersion: "18945" uid: 10755b63-fb3d-44aa-bfa9-c4d8b726455d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-0b4fc4-predictor namespace: kserve-ci-e2e-test resourceVersion: "18931" uid: 73f2b6ba-d68a-4050-b3be-6e458c2533f7 kind: Event lastTimestamp: "2026-06-02T16:05:14Z" message: Scaled up replica set isvc-init-fail-0b4fc4-predictor-9466c49c6 from 0 to 1 metadata: creationTimestamp: "2026-06-02T16:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:05:14Z" name: isvc-init-fail-0b4fc4-predictor.18b54f345ca7759a namespace: kserve-ci-e2e-test resourceVersion: "18936" uid: baff9182-68a2-4a1c-906a-a7e38f0c4067 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-06-02T16:05:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-0b4fc4 namespace: kserve-ci-e2e-test resourceVersion: "18905" uid: 4cd078d7-3078-4111-b5f8-8a49da0ae2f5 kind: Event lastTimestamp: "2026-06-02T16:05:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-02T16:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:05:20Z" name: isvc-init-fail-0b4fc4.18b54f345c835fd3 namespace: kserve-ci-e2e-test resourceVersion: "19027" uid: be26f575-bd1c-4d44-a341-56a77d1bb85b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-0b4fc4 namespace: kserve-ci-e2e-test resourceVersion: "18905" uid: 4cd078d7-3078-4111-b5f8-8a49da0ae2f5 kind: Event lastTimestamp: "2026-06-02T16:05:14Z" message: 'Failed to update status for InferenceService "isvc-init-fail-0b4fc4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-0b4fc4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-02T16:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:05:14Z" name: isvc-init-fail-0b4fc4.18b54f345d04afaa namespace: kserve-ci-e2e-test resourceVersion: "18938" uid: d7f50c27-2422-4dee-9e58-44c408df4202 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-0b4fc4 namespace: kserve-ci-e2e-test resourceVersion: "18905" uid: 4cd078d7-3078-4111-b5f8-8a49da0ae2f5 kind: Event lastTimestamp: "2026-06-02T16:05:14Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-0b4fc4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-02T16:05:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:05:14Z" name: isvc-init-fail-0b4fc4.18b54f345d04ff94 namespace: kserve-ci-e2e-test resourceVersion: "18940" uid: fecda342-c4f1-4d4c-b76a-8dab9fbe9ea7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-06-02T15:59:50.118296Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: 84534d3c-6555-42a5-b308-9d8a82670acd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc to ip-10-0-140-246.ec2.internal metadata: creationTimestamp: "2026-06-02T15:59:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-02T15:59:50Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc.18b54ee8f36d8a41 namespace: kserve-ci-e2e-test resourceVersion: "16239" uid: e422bbb9-c39d-4ea0-988f-c2bc817a0b83 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8ddcd7f6c-hs6d2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc namespace: kserve-ci-e2e-test resourceVersion: "16245" uid: 84534d3c-6555-42a5-b308-9d8a82670acd kind: Event lastTimestamp: "2026-06-02T15:59:50Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-02T15:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-02T15:59:50Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc.18b54ee90cc752b3 namespace: kserve-ci-e2e-test resourceVersion: "16253" uid: 25b929ac-615a-4661-ab88-cec8e364ad35 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 84534d3c-6555-42a5-b308-9d8a82670acd kind: Event lastTimestamp: "2026-06-02T15:59:50Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ca9e7e1d07975dffacc4b293d655f2a02df4615629916a1cdc022036c5a27a57" already present on machine metadata: creationTimestamp: "2026-06-02T15:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:50Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc.18b54ee90f30613f namespace: kserve-ci-e2e-test resourceVersion: "16255" uid: c262df6e-ca0e-4e3e-ad0f-e0f9e88b0993 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 84534d3c-6555-42a5-b308-9d8a82670acd kind: Event lastTimestamp: "2026-06-02T15:59:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-02T15:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:50Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc.18b54ee91231201f namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: 7837ca82-b2ae-47ca-990f-306fe5b6a607 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 84534d3c-6555-42a5-b308-9d8a82670acd kind: Event lastTimestamp: "2026-06-02T15:59:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-02T15:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:50Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc.18b54ee9128dd721 namespace: kserve-ci-e2e-test resourceVersion: "16257" uid: 83846acb-0fe9-4817-89f0-08e963e91c39 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 84534d3c-6555-42a5-b308-9d8a82670acd kind: Event lastTimestamp: "2026-06-02T15:59:54Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1560" already present on machine metadata: creationTimestamp: "2026-06-02T15:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:54Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc.18b54ee9fcd467e1 namespace: kserve-ci-e2e-test resourceVersion: "16294" uid: 19e7393a-7175-40ae-9568-93ae8f0b1b57 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 84534d3c-6555-42a5-b308-9d8a82670acd kind: Event lastTimestamp: "2026-06-02T15:59:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-02T15:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:54Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc.18b54ee9ff6ec81b namespace: kserve-ci-e2e-test resourceVersion: "16296" uid: b46c0d3c-b0c3-4988-9642-65ef160b295e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 84534d3c-6555-42a5-b308-9d8a82670acd kind: Event lastTimestamp: "2026-06-02T15:59:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-02T15:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:54Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc.18b54ee9ffd1c21c namespace: kserve-ci-e2e-test resourceVersion: "16297" uid: eb480945-cca3-4017-a0a0-cead06b029f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 84534d3c-6555-42a5-b308-9d8a82670acd kind: Event lastTimestamp: "2026-06-02T15:59:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-02T15:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:54Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc.18b54ee9ffdcbe44 namespace: kserve-ci-e2e-test resourceVersion: "16298" uid: 6b27c670-6b56-4156-a3bc-ce959631aad7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 84534d3c-6555-42a5-b308-9d8a82670acd kind: Event lastTimestamp: "2026-06-02T15:59:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-02T15:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:54Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc.18b54eea02ce2d71 namespace: kserve-ci-e2e-test resourceVersion: "16301" uid: c0e4e275-f49d-4982-b659-30b55af3261d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 84534d3c-6555-42a5-b308-9d8a82670acd kind: Event lastTimestamp: "2026-06-02T15:59:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T15:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:54Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc.18b54eea0358d7ec namespace: kserve-ci-e2e-test resourceVersion: "16302" uid: 02b0f977-8237-4912-8022-16a659c10e94 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 84534d3c-6555-42a5-b308-9d8a82670acd kind: Event lastTimestamp: "2026-06-02T15:59:54Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:4411ca57686833cde23c90dc27bf5e4bed3f508dac886872826c4a0995fadff0" already present on machine metadata: creationTimestamp: "2026-06-02T15:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:54Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc.18b54eea0363994f namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 875d1647-1905-40d7-815b-3c839a3a26bd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 84534d3c-6555-42a5-b308-9d8a82670acd kind: Event lastTimestamp: "2026-06-02T15:59:54Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-06-02T15:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:54Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc.18b54eea06046e6f namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: 6052b8ce-578a-49ed-8970-8eb874761d8b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 84534d3c-6555-42a5-b308-9d8a82670acd kind: Event lastTimestamp: "2026-06-02T15:59:54Z" message: Started container agent metadata: creationTimestamp: "2026-06-02T15:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:54Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc.18b54eea0678cc82 namespace: kserve-ci-e2e-test resourceVersion: "16306" uid: 7ac41ea9-5ead-4d7c-8daf-f8d6797c0276 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-06-02T15:59:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 84534d3c-6555-42a5-b308-9d8a82670acd kind: Event lastTimestamp: "2026-06-02T16:01:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-02T15:59:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:01:32Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc.18b54eea38bba023 namespace: kserve-ci-e2e-test resourceVersion: "17139" uid: 1dc0019c-618c-4064-86f4-8f6ba322bbdb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-06-02T15:59:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 84534d3c-6555-42a5-b308-9d8a82670acd kind: Event lastTimestamp: "2026-06-02T16:01:32Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-06-02T15:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:01:32Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc.18b54eea74a13bf4 namespace: kserve-ci-e2e-test resourceVersion: "17140" uid: d6c79524-6c40-4687-8a0d-36aea7109f2c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 84534d3c-6555-42a5-b308-9d8a82670acd kind: Event lastTimestamp: "2026-06-02T16:01:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-02T16:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:01:15Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc.18b54efcc59b3ca9 namespace: kserve-ci-e2e-test resourceVersion: "16941" uid: 2dc92b84-e69c-4710-a54c-1202af428300 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 84534d3c-6555-42a5-b308-9d8a82670acd kind: Event lastTimestamp: "2026-06-02T16:01:15Z" message: Stopping container agent metadata: creationTimestamp: "2026-06-02T16:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:01:15Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc.18b54efcc59b8ef6 namespace: kserve-ci-e2e-test resourceVersion: "16943" uid: cd659c62-3662-4c53-8030-adb60bdf2aa9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 84534d3c-6555-42a5-b308-9d8a82670acd kind: Event lastTimestamp: "2026-06-02T16:01:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T16:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:01:15Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc.18b54efcc59bfbc9 namespace: kserve-ci-e2e-test resourceVersion: "16944" uid: 98cd1d56-da7a-40d2-bd7a-95b7f2e2aab1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-02T16:01:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 84534d3c-6555-42a5-b308-9d8a82670acd kind: Event lastTimestamp: "2026-06-02T16:01:32Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-02T16:01:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:01:32Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc.18b54efd50c83071 namespace: kserve-ci-e2e-test resourceVersion: "17138" uid: fc3f0e32-d445-40cf-9bb9-576db6cded5a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-0d859-predictor-7cf485cd8c namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 8c568db1-98a2-4d6d-8e37-c0c8c5ddd0e8 kind: Event lastTimestamp: "2026-06-02T15:59:50Z" message: 'Created pod: isvc-logger-raw-0d859-predictor-7cf485cd8c-8kvfc' metadata: creationTimestamp: "2026-06-02T15:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:59:50Z" name: isvc-logger-raw-0d859-predictor-7cf485cd8c.18b54ee8f25c4b5b namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 2bb6e877-ad22-44ef-80da-1c4dc7cb896d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-0d859-predictor namespace: kserve-ci-e2e-test resourceVersion: "16222" uid: 7fac844e-d267-4223-8031-82727a3eb576 kind: Event lastTimestamp: "2026-06-02T15:59:49Z" message: Scaled up replica set isvc-logger-raw-0d859-predictor-7cf485cd8c from 0 to 1 metadata: creationTimestamp: "2026-06-02T15:59:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:59:49Z" name: isvc-logger-raw-0d859-predictor.18b54ee8ec0a015b namespace: kserve-ci-e2e-test resourceVersion: "16226" uid: cc54cb7f-e384-4551-b666-eb9eb376a175 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T16:00:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-0d859-predictor namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: d928d54b-71b2-4a02-8e85-09fba610fdf5 kind: Event lastTimestamp: "2026-06-02T16:00:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T16:00:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:00:35Z" name: isvc-logger-raw-0d859-predictor.18b54eec6bba993a namespace: kserve-ci-e2e-test resourceVersion: "16607" uid: 37f46aa6-3f1b-4811-8a21-84630a97ecd0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T16:00:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-0d859-predictor namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: d928d54b-71b2-4a02-8e85-09fba610fdf5 kind: Event lastTimestamp: "2026-06-02T16:00:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T16:00:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:00:35Z" name: isvc-logger-raw-0d859-predictor.18b54eec6c43bd87 namespace: kserve-ci-e2e-test resourceVersion: "16609" uid: d1b8b64e-b5e5-44f4-b480-2c3cfc8db464 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-02T16:00:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-0d859-predictor namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: d928d54b-71b2-4a02-8e85-09fba610fdf5 kind: Event lastTimestamp: "2026-06-02T16:01:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-02T16:00:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:01:05Z" name: isvc-logger-raw-0d859-predictor.18b54eefebc37bc8 namespace: kserve-ci-e2e-test resourceVersion: "16829" uid: 58983251-a000-4132-bbb0-e8204b010d58 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-02T16:00:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-0d859-predictor namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: d928d54b-71b2-4a02-8e85-09fba610fdf5 kind: Event lastTimestamp: "2026-06-02T16:01:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-02T16:00:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:01:05Z" name: isvc-logger-raw-0d859-predictor.18b54eefec33e604 namespace: kserve-ci-e2e-test resourceVersion: "16830" uid: aa88d516-49b3-44b2-b44c-a01c339c784e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-06-02T15:59:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-0d859 namespace: kserve-ci-e2e-test resourceVersion: "16212" uid: 9ccdb41f-af14-46bb-954e-5e3295e4f389 kind: Event lastTimestamp: "2026-06-02T16:01:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-02T15:59:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:01:02Z" name: isvc-logger-raw-0d859.18b54ee8ebfac932 namespace: kserve-ci-e2e-test resourceVersion: "16811" uid: 865f9003-644b-4b81-975d-4fd0c40aca02 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T15:59:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-0d859 namespace: kserve-ci-e2e-test resourceVersion: "16212" uid: 9ccdb41f-af14-46bb-954e-5e3295e4f389 kind: Event lastTimestamp: "2026-06-02T15:59:50Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-0d859": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-0d859": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-02T15:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:59:50Z" name: isvc-logger-raw-0d859.18b54ee8ecd94515 namespace: kserve-ci-e2e-test resourceVersion: "16248" uid: c2b14ef1-10ac-4bf8-93b5-7f7311d9b086 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-0d859 namespace: kserve-ci-e2e-test resourceVersion: "16213" uid: 9ccdb41f-af14-46bb-954e-5e3295e4f389 kind: Event lastTimestamp: "2026-06-02T15:59:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-0d859": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-02T15:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:59:50Z" name: isvc-logger-raw-0d859.18b54ee8f7ab9ab9 namespace: kserve-ci-e2e-test resourceVersion: "16249" uid: 5912eeb0-11a0-441c-82c5-b4af685b3656 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-0d859 namespace: kserve-ci-e2e-test resourceVersion: "16809" uid: 9ccdb41f-af14-46bb-954e-5e3295e4f389 kind: Event lastTimestamp: "2026-06-02T16:01:02Z" message: InferenceService [isvc-logger-raw-0d859] is Ready metadata: creationTimestamp: "2026-06-02T16:01:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:01:02Z" name: isvc-logger-raw-0d859.18b54ef9d8b11df3 namespace: kserve-ci-e2e-test resourceVersion: "16810" uid: 83342c71-2061-4a9e-9b89-e76e400560d2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-02T16:03:35.837089Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44 namespace: kserve-ci-e2e-test resourceVersion: "18066" uid: d7b6b9d3-a7bf-4793-8879-4dc2e14973a3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-cffc11-predictor-6846fdc6ff-k4x44 to ip-10-0-140-246.ec2.internal metadata: creationTimestamp: "2026-06-02T16:03:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-02T16:03:35Z" name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44.18b54f1d8150c4de namespace: kserve-ci-e2e-test resourceVersion: "18070" uid: d1fbf5c2-e365-4d4c-b60c-8d0a5ecc654a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8ddcd7f6c-hs6d2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:03:36Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44 namespace: kserve-ci-e2e-test resourceVersion: "18074" uid: d7b6b9d3-a7bf-4793-8879-4dc2e14973a3 kind: Event lastTimestamp: "2026-06-02T16:03:36Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-02T16:03:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-02T16:03:36Z" name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44.18b54f1d9abfea5d namespace: kserve-ci-e2e-test resourceVersion: "18083" uid: 3eb5065e-b6f3-4d29-a830-0d2f016e2137 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:03:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44 namespace: kserve-ci-e2e-test resourceVersion: "18068" uid: d7b6b9d3-a7bf-4793-8879-4dc2e14973a3 kind: Event lastTimestamp: "2026-06-02T16:03:36Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ca9e7e1d07975dffacc4b293d655f2a02df4615629916a1cdc022036c5a27a57" already present on machine metadata: creationTimestamp: "2026-06-02T16:03:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:03:36Z" name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44.18b54f1d9d890544 namespace: kserve-ci-e2e-test resourceVersion: "18085" uid: 10912c18-2202-4c41-a6a4-115f47bba54b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:03:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44 namespace: kserve-ci-e2e-test resourceVersion: "18068" uid: d7b6b9d3-a7bf-4793-8879-4dc2e14973a3 kind: Event lastTimestamp: "2026-06-02T16:03:36Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-02T16:03:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:03:36Z" name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44.18b54f1da08a6351 namespace: kserve-ci-e2e-test resourceVersion: "18086" uid: 79ae8b0c-5353-4321-97d8-0636375c902f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:03:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44 namespace: kserve-ci-e2e-test resourceVersion: "18068" uid: d7b6b9d3-a7bf-4793-8879-4dc2e14973a3 kind: Event lastTimestamp: "2026-06-02T16:03:36Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-02T16:03:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:03:36Z" name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44.18b54f1da0f755f3 namespace: kserve-ci-e2e-test resourceVersion: "18088" uid: f7ff93dd-838c-4ef3-8e19-b7e237048972 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:03:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44 namespace: kserve-ci-e2e-test resourceVersion: "18068" uid: d7b6b9d3-a7bf-4793-8879-4dc2e14973a3 kind: Event lastTimestamp: "2026-06-02T16:03:40Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1560" already present on machine metadata: creationTimestamp: "2026-06-02T16:03:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:03:40Z" name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44.18b54f1e891025fa namespace: kserve-ci-e2e-test resourceVersion: "18121" uid: d61652dd-14a1-4730-ae7a-5f2ec9903a13 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:03:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44 namespace: kserve-ci-e2e-test resourceVersion: "18068" uid: d7b6b9d3-a7bf-4793-8879-4dc2e14973a3 kind: Event lastTimestamp: "2026-06-02T16:03:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-02T16:03:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:03:40Z" name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44.18b54f1e8c0edb4e namespace: kserve-ci-e2e-test resourceVersion: "18123" uid: 379e1dcf-63e1-41b0-bbe4-758de062332e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:03:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44 namespace: kserve-ci-e2e-test resourceVersion: "18068" uid: d7b6b9d3-a7bf-4793-8879-4dc2e14973a3 kind: Event lastTimestamp: "2026-06-02T16:03:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-02T16:03:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:03:40Z" name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44.18b54f1e8c80e1dd namespace: kserve-ci-e2e-test resourceVersion: "18124" uid: d1f23112-1a81-4d84-9f17-cf89c9fde88d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:03:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44 namespace: kserve-ci-e2e-test resourceVersion: "18068" uid: d7b6b9d3-a7bf-4793-8879-4dc2e14973a3 kind: Event lastTimestamp: "2026-06-02T16:03:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-02T16:03:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:03:40Z" name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44.18b54f1e8c8bb0be namespace: kserve-ci-e2e-test resourceVersion: "18125" uid: 61d11939-9e75-435b-b567-26d098708e69 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:03:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44 namespace: kserve-ci-e2e-test resourceVersion: "18068" uid: d7b6b9d3-a7bf-4793-8879-4dc2e14973a3 kind: Event lastTimestamp: "2026-06-02T16:03:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-02T16:03:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:03:40Z" name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44.18b54f1e8f698ee6 namespace: kserve-ci-e2e-test resourceVersion: "18128" uid: 08acb62c-6455-400e-95f0-6cbee4a8cffb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:03:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44 namespace: kserve-ci-e2e-test resourceVersion: "18068" uid: d7b6b9d3-a7bf-4793-8879-4dc2e14973a3 kind: Event lastTimestamp: "2026-06-02T16:03:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T16:03:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:03:40Z" name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44.18b54f1e8fdb4fae namespace: kserve-ci-e2e-test resourceVersion: "18130" uid: 9c42797f-700e-4902-85f4-800123371015 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-06-02T16:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44 namespace: kserve-ci-e2e-test resourceVersion: "18068" uid: d7b6b9d3-a7bf-4793-8879-4dc2e14973a3 kind: Event lastTimestamp: "2026-06-02T16:05:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-02T16:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:18Z" name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44.18b54f1f00ec9e71 namespace: kserve-ci-e2e-test resourceVersion: "19000" uid: 63a334ae-1902-4748-b531-ed115a658fc6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44 namespace: kserve-ci-e2e-test resourceVersion: "18068" uid: d7b6b9d3-a7bf-4793-8879-4dc2e14973a3 kind: Event lastTimestamp: "2026-06-02T16:05:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-02T16:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:13Z" name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44.18b54f3454e2f351 namespace: kserve-ci-e2e-test resourceVersion: "18903" uid: 85630eff-27cb-4d3f-9efb-069f60149bf5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44 namespace: kserve-ci-e2e-test resourceVersion: "18068" uid: d7b6b9d3-a7bf-4793-8879-4dc2e14973a3 kind: Event lastTimestamp: "2026-06-02T16:05:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T16:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:13Z" name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44.18b54f3454e37698 namespace: kserve-ci-e2e-test resourceVersion: "18904" uid: 89397a4c-a5cd-41d6-b3c0-45c0fc92d3ac reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44 namespace: kserve-ci-e2e-test resourceVersion: "18068" uid: d7b6b9d3-a7bf-4793-8879-4dc2e14973a3 kind: Event lastTimestamp: "2026-06-02T16:05:18Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-02T16:05:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:18Z" name: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44.18b54f355b28e62d namespace: kserve-ci-e2e-test resourceVersion: "18999" uid: e07c1843-1dbf-431b-974a-3af104c17557 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:03:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-cffc11-predictor-6846fdc6ff namespace: kserve-ci-e2e-test resourceVersion: "18056" uid: 54441a03-7376-4f88-8fe1-08daa9cb2f35 kind: Event lastTimestamp: "2026-06-02T16:03:35Z" message: 'Created pod: isvc-primary-cffc11-predictor-6846fdc6ff-k4x44' metadata: creationTimestamp: "2026-06-02T16:03:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:03:35Z" name: isvc-primary-cffc11-predictor-6846fdc6ff.18b54f1d80a8423a namespace: kserve-ci-e2e-test resourceVersion: "18067" uid: af120a98-b62a-419d-a40e-ae42808b55df reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:03:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-cffc11-predictor namespace: kserve-ci-e2e-test resourceVersion: "18052" uid: 9b502126-6138-46e6-96d0-d6004efc2936 kind: Event lastTimestamp: "2026-06-02T16:03:35Z" message: Scaled up replica set isvc-primary-cffc11-predictor-6846fdc6ff from 0 to 1 metadata: creationTimestamp: "2026-06-02T16:03:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:03:35Z" name: isvc-primary-cffc11-predictor.18b54f1d79f7b816 namespace: kserve-ci-e2e-test resourceVersion: "18059" uid: ae620320-8171-4f22-b59b-8c4ef3b1d6eb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T16:03:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-cffc11-predictor namespace: kserve-ci-e2e-test resourceVersion: "18055" uid: c426ba1f-8b74-4794-9f59-3e4abf771475 kind: Event lastTimestamp: "2026-06-02T16:04:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T16:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:04:20Z" name: isvc-primary-cffc11-predictor.18b54f20fa3891ba namespace: kserve-ci-e2e-test resourceVersion: "18428" uid: 92741749-01e9-4b21-be24-bc96fb75f988 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T16:03:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-cffc11-predictor namespace: kserve-ci-e2e-test resourceVersion: "18055" uid: c426ba1f-8b74-4794-9f59-3e4abf771475 kind: Event lastTimestamp: "2026-06-02T16:04:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T16:03:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:04:20Z" name: isvc-primary-cffc11-predictor.18b54f20fb371cb0 namespace: kserve-ci-e2e-test resourceVersion: "18429" uid: 8f48b787-9405-4942-8830-4fb1710df025 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-02T16:04:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-cffc11-predictor namespace: kserve-ci-e2e-test resourceVersion: "18212" uid: c426ba1f-8b74-4794-9f59-3e4abf771475 kind: Event lastTimestamp: "2026-06-02T16:05:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-02T16:04:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:05:05Z" name: isvc-primary-cffc11-predictor.18b54f247a6f31a7 namespace: kserve-ci-e2e-test resourceVersion: "18816" uid: 5c701b6b-dace-47b0-83be-b9b6e66dea02 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-02T16:04:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-cffc11-predictor namespace: kserve-ci-e2e-test resourceVersion: "18212" uid: c426ba1f-8b74-4794-9f59-3e4abf771475 kind: Event lastTimestamp: "2026-06-02T16:05:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-02T16:04:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:05:05Z" name: isvc-primary-cffc11-predictor.18b54f247b020dfe namespace: kserve-ci-e2e-test resourceVersion: "18817" uid: 54f8fec7-948c-4789-9c63-c9ccf4b5cf5c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-06-02T16:03:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-cffc11 namespace: kserve-ci-e2e-test resourceVersion: "18024" uid: 59b95e51-0b29-4c7d-9a13-9c98932321a5 kind: Event lastTimestamp: "2026-06-02T16:04:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-02T16:03:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:04:48Z" name: isvc-primary-cffc11.18b54f1d79d63c1e namespace: kserve-ci-e2e-test resourceVersion: "18631" uid: 79fef3a3-de63-416e-aa00-06e67648acbb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T16:03:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-cffc11 namespace: kserve-ci-e2e-test resourceVersion: "18024" uid: 59b95e51-0b29-4c7d-9a13-9c98932321a5 kind: Event lastTimestamp: "2026-06-02T16:03:35Z" message: 'Failed to update status for InferenceService "isvc-primary-cffc11": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-cffc11": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-02T16:03:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:03:35Z" name: isvc-primary-cffc11.18b54f1d7afaaccd namespace: kserve-ci-e2e-test resourceVersion: "18076" uid: 52a45393-c2eb-4feb-b0bd-204b3b91ee5d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T16:03:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-cffc11 namespace: kserve-ci-e2e-test resourceVersion: "18024" uid: 59b95e51-0b29-4c7d-9a13-9c98932321a5 kind: Event lastTimestamp: "2026-06-02T16:03:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-cffc11": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-02T16:03:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:03:35Z" name: isvc-primary-cffc11.18b54f1d7afaf2dc namespace: kserve-ci-e2e-test resourceVersion: "18077" uid: 16bbd2c2-e4b8-4c01-82cf-8a3d0baa5c3e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:04:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-cffc11 namespace: kserve-ci-e2e-test resourceVersion: "18628" uid: 59b95e51-0b29-4c7d-9a13-9c98932321a5 kind: Event lastTimestamp: "2026-06-02T16:04:48Z" message: InferenceService [isvc-primary-cffc11] is Ready metadata: creationTimestamp: "2026-06-02T16:04:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:04:48Z" name: isvc-primary-cffc11.18b54f2e651fe9e7 namespace: kserve-ci-e2e-test resourceVersion: "18630" uid: 62d1100f-aba7-4e44-95db-8bb8b7aac3c4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-02T15:54:14.146038Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13137" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb to ip-10-0-140-246.ec2.internal metadata: creationTimestamp: "2026-06-02T15:54:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-02T15:54:14Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54e9ab9eb69ff namespace: kserve-ci-e2e-test resourceVersion: "13141" uid: ed2168a0-41d7-4b50-a3f7-4831d2c08127 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8ddcd7f6c-hs6d2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:54:14Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13147" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:54:14Z" message: Add eth0 [10.134.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-02T15:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-02T15:54:14Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54e9ad328eb9b namespace: kserve-ci-e2e-test resourceVersion: "13154" uid: 2792416d-ee8f-437d-8e51-13945739d671 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:54:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:54:14Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:ca9e7e1d07975dffacc4b293d655f2a02df4615629916a1cdc022036c5a27a57" metadata: creationTimestamp: "2026-06-02T15:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:54:14Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54e9ad550480a namespace: kserve-ci-e2e-test resourceVersion: "13156" uid: 55cf9cb8-655d-45ae-868b-0a91b49fa3bf reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:54:18Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:ca9e7e1d07975dffacc4b293d655f2a02df4615629916a1cdc022036c5a27a57" in 4.332s (4.333s including waiting). Image size: 301469622 bytes.' metadata: creationTimestamp: "2026-06-02T15:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:54:18Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54e9bd794f88f namespace: kserve-ci-e2e-test resourceVersion: "13184" uid: 67e891ed-6d0f-4127-b91a-1e17daa5d360 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:54:18Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-02T15:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:54:18Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54e9bdacb4665 namespace: kserve-ci-e2e-test resourceVersion: "13185" uid: 2fac71b6-11c3-4c1c-963d-9c63db318055 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:54:18Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-02T15:54:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:54:19Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54e9bdb2f595c namespace: kserve-ci-e2e-test resourceVersion: "13186" uid: 463d3e4c-0003-475e-bf97-c7a9d1b58606 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:54:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:54:23Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1560" metadata: creationTimestamp: "2026-06-02T15:54:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:54:23Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54e9ce7b9453c namespace: kserve-ci-e2e-test resourceVersion: "13228" uid: 760b7a29-8bc6-4782-9889-e462eddb79e0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:54:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:54:38Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1560" in 14.506s (14.506s including waiting). Image size: 1560927157 bytes.' metadata: creationTimestamp: "2026-06-02T15:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:54:38Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54ea0485cbaed namespace: kserve-ci-e2e-test resourceVersion: "13329" uid: a7c3a39f-22e2-4cbd-b4e9-de9cf2716070 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:54:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:54:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-02T15:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:54:38Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54ea04b5ff1f8 namespace: kserve-ci-e2e-test resourceVersion: "13330" uid: 321fa61f-732d-4b02-92fd-d1d3e10288fa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:54:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:54:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-02T15:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:54:38Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54ea04bca6ea0 namespace: kserve-ci-e2e-test resourceVersion: "13331" uid: d4a1f6fc-e6dd-421a-b703-f251b8a9ddf8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:54:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:54:38Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-06-02T15:54:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:54:38Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54ea04bd68bd2 namespace: kserve-ci-e2e-test resourceVersion: "13332" uid: 46528bec-7b44-4a1a-845f-6de6ddb48a7f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:54:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:54:40Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.137s (2.137s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-06-02T15:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:54:40Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54ea0cb3d3081 namespace: kserve-ci-e2e-test resourceVersion: "13348" uid: a3551ca1-cb29-4ae3-94e8-9b94f67ed7b2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:54:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:54:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-02T15:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:54:40Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54ea0ce3f0ae6 namespace: kserve-ci-e2e-test resourceVersion: "13349" uid: 6a8d49ce-baa1-47d7-b6a2-b7a5c04a7e74 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:54:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:54:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T15:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:54:40Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54ea0cec92583 namespace: kserve-ci-e2e-test resourceVersion: "13350" uid: 0d1a7344-e9af-4e64-a42a-71e29008afac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:54:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:54:40Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:4411ca57686833cde23c90dc27bf5e4bed3f508dac886872826c4a0995fadff0" metadata: creationTimestamp: "2026-06-02T15:54:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:54:40Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54ea0ced5a4ab namespace: kserve-ci-e2e-test resourceVersion: "13351" uid: d68875ed-860b-4e7c-8628-2331a46a7786 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:54:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:54:44Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:4411ca57686833cde23c90dc27bf5e4bed3f508dac886872826c4a0995fadff0" in 3.797s (3.797s including waiting). Image size: 239695449 bytes.' metadata: creationTimestamp: "2026-06-02T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:54:44Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54ea1b12da386 namespace: kserve-ci-e2e-test resourceVersion: "13380" uid: fdcbbe27-6648-456e-89b5-4f4e121d42c4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:54:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:54:44Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-06-02T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:54:44Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54ea1b591bacc namespace: kserve-ci-e2e-test resourceVersion: "13382" uid: 845a7f1d-3ad1-47b7-aa85-a796fc79a59a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:54:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:54:44Z" message: Started container agent metadata: creationTimestamp: "2026-06-02T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:54:44Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54ea1b5f27ae0 namespace: kserve-ci-e2e-test resourceVersion: "13383" uid: 168ac5b0-5078-4697-a3bf-e984d7237633 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-06-02T15:54:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:56:17Z" message: 'Readiness probe failed: dial tcp 10.134.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-02T15:54:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:56:17Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54ea1d0717408 namespace: kserve-ci-e2e-test resourceVersion: "14207" uid: 6e474213-84f1-4fe4-ade4-db71ef494442 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-06-02T15:54:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:56:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-06-02T15:54:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:56:17Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54ea20c30a349 namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: 940aeb7d-be0e-4edd-8caf-c3deabced836 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:55:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:55:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-02T15:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:55:58Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54eb31eef51d1 namespace: kserve-ci-e2e-test resourceVersion: "13953" uid: d8634500-b329-491b-9edd-cd93c44839df reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:55:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:55:58Z" message: Stopping container agent metadata: creationTimestamp: "2026-06-02T15:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:55:58Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54eb31eef8962 namespace: kserve-ci-e2e-test resourceVersion: "13956" uid: cb58acc1-70f6-44fa-9486-148e446a954a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:55:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:55:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T15:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:55:58Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54eb31ef00f64 namespace: kserve-ci-e2e-test resourceVersion: "13959" uid: 7af1d56c-5463-4dad-bed2-f3a006517c95 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-02T15:56:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb namespace: kserve-ci-e2e-test resourceVersion: "13139" uid: 75a18a34-f0af-43bd-bcf3-82f5c8623661 kind: Event lastTimestamp: "2026-06-02T15:56:15Z" message: 'Readiness probe failed: Get "https://10.134.0.19:8643/healthz": dial tcp 10.134.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-02T15:56:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:56:15Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb.18b54eb38282b7fc namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 386d757a-0671-4a60-9c19-61b60da61917 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:54:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b namespace: kserve-ci-e2e-test resourceVersion: "13121" uid: 06464516-d2b1-4ce4-a35e-bd926f8ec9d6 kind: Event lastTimestamp: "2026-06-02T15:54:14Z" message: 'Created pod: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b-q6smb' metadata: creationTimestamp: "2026-06-02T15:54:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:54:14Z" name: isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b.18b54e9ab888434b namespace: kserve-ci-e2e-test resourceVersion: "13138" uid: 1e058b44-0450-4164-81b5-513937e44477 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:54:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-b5790-predictor namespace: kserve-ci-e2e-test resourceVersion: "13120" uid: 4f76d395-fb51-448a-8233-d0b284cde6a3 kind: Event lastTimestamp: "2026-06-02T15:54:13Z" message: Scaled up replica set isvc-raw-sklearn-batcher-b5790-predictor-58fd77887b from 0 to 1 metadata: creationTimestamp: "2026-06-02T15:54:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:54:13Z" name: isvc-raw-sklearn-batcher-b5790-predictor.18b54e9aa77f35b1 namespace: kserve-ci-e2e-test resourceVersion: "13122" uid: 5d373fe0-63e3-45f4-8327-e6b790f1fb3e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-02T15:54:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-b5790-predictor namespace: kserve-ci-e2e-test resourceVersion: "13127" uid: 488837ab-5a9e-4f8f-9f60-a33db650a001 kind: Event lastTimestamp: "2026-06-02T15:55:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T15:54:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:55:14Z" name: isvc-raw-sklearn-batcher-b5790-predictor.18b54e9e2d737a45 namespace: kserve-ci-e2e-test resourceVersion: "13601" uid: 148e0d09-8914-485d-9d05-0a60f88881e0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-02T15:54:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-b5790-predictor namespace: kserve-ci-e2e-test resourceVersion: "13127" uid: 488837ab-5a9e-4f8f-9f60-a33db650a001 kind: Event lastTimestamp: "2026-06-02T15:55:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T15:54:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:55:14Z" name: isvc-raw-sklearn-batcher-b5790-predictor.18b54e9e2de9d306 namespace: kserve-ci-e2e-test resourceVersion: "13602" uid: d5a0f365-aeda-45ca-8be7-751c1ae4bd0e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T15:55:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-b5790-predictor namespace: kserve-ci-e2e-test resourceVersion: "13267" uid: 488837ab-5a9e-4f8f-9f60-a33db650a001 kind: Event lastTimestamp: "2026-06-02T15:55:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-02T15:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:55:44Z" name: isvc-raw-sklearn-batcher-b5790-predictor.18b54eac2ab4b33d namespace: kserve-ci-e2e-test resourceVersion: "13817" uid: f1dc8cd9-8ad4-4c18-9b88-8ae84059426e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T15:55:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-b5790-predictor namespace: kserve-ci-e2e-test resourceVersion: "13267" uid: 488837ab-5a9e-4f8f-9f60-a33db650a001 kind: Event lastTimestamp: "2026-06-02T15:55:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-02T15:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:55:44Z" name: isvc-raw-sklearn-batcher-b5790-predictor.18b54eac2b280a4a namespace: kserve-ci-e2e-test resourceVersion: "13818" uid: dd6b5fb6-2e5d-4244-8f5c-4f805acddfde reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-06-02T15:54:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-b5790 namespace: kserve-ci-e2e-test resourceVersion: "13107" uid: a607d01e-3c8c-4a4d-9210-1dd58c7431eb kind: Event lastTimestamp: "2026-06-02T15:55:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-02T15:54:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:55:47Z" name: isvc-raw-sklearn-batcher-b5790.18b54e9aae02955d namespace: kserve-ci-e2e-test resourceVersion: "13855" uid: b6ffd618-a80f-4f2c-b176-1a4c0d814263 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:54:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-b5790 namespace: kserve-ci-e2e-test resourceVersion: "13107" uid: a607d01e-3c8c-4a4d-9210-1dd58c7431eb kind: Event lastTimestamp: "2026-06-02T15:54:13Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-b5790": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-b5790": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-02T15:54:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:54:13Z" name: isvc-raw-sklearn-batcher-b5790.18b54e9aaea4b03f namespace: kserve-ci-e2e-test resourceVersion: "13129" uid: 7b3752d8-0ce0-4141-8d38-82f9e67779e2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:55:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-b5790 namespace: kserve-ci-e2e-test resourceVersion: "13853" uid: a607d01e-3c8c-4a4d-9210-1dd58c7431eb kind: Event lastTimestamp: "2026-06-02T15:55:47Z" message: InferenceService [isvc-raw-sklearn-batcher-b5790] is Ready metadata: creationTimestamp: "2026-06-02T15:55:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:55:47Z" name: isvc-raw-sklearn-batcher-b5790.18b54eb086541ee9 namespace: kserve-ci-e2e-test resourceVersion: "13854" uid: 207c7492-301e-4787-9a1a-c955e25ef52b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-02T16:04:55.805046Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-cffc11-predictor-85fd598f88-gbsnp namespace: kserve-ci-e2e-test resourceVersion: "18711" uid: 254dbfa2-2250-4019-a29c-78089b23f519 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-cffc11-predictor-85fd598f88-gbsnp to ip-10-0-140-246.ec2.internal metadata: creationTimestamp: "2026-06-02T16:04:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-02T16:04:55Z" name: isvc-secondary-cffc11-predictor-85fd598f88-gbsnp.18b54f301fc6ef25 namespace: kserve-ci-e2e-test resourceVersion: "18715" uid: 3e70ea48-84e5-4d2d-bfc7-fd7db1771f44 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8ddcd7f6c-hs6d2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:04:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-cffc11-predictor-85fd598f88-gbsnp namespace: kserve-ci-e2e-test resourceVersion: "18719" uid: 254dbfa2-2250-4019-a29c-78089b23f519 kind: Event lastTimestamp: "2026-06-02T16:04:56Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-02T16:04:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-02T16:04:56Z" name: isvc-secondary-cffc11-predictor-85fd598f88-gbsnp.18b54f3039ed1c1c namespace: kserve-ci-e2e-test resourceVersion: "18727" uid: 093da989-9948-418f-a119-8edcd659f734 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T16:04:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-cffc11-predictor-85fd598f88-gbsnp namespace: kserve-ci-e2e-test resourceVersion: "18714" uid: 254dbfa2-2250-4019-a29c-78089b23f519 kind: Event lastTimestamp: "2026-06-02T16:05:02Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ca9e7e1d07975dffacc4b293d655f2a02df4615629916a1cdc022036c5a27a57" already present on machine metadata: creationTimestamp: "2026-06-02T16:04:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:02Z" name: isvc-secondary-cffc11-predictor-85fd598f88-gbsnp.18b54f303ba123c9 namespace: kserve-ci-e2e-test resourceVersion: "18778" uid: 44bbfbb3-790e-4e3e-aa7d-95f276e79712 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T16:04:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-cffc11-predictor-85fd598f88-gbsnp namespace: kserve-ci-e2e-test resourceVersion: "18714" uid: 254dbfa2-2250-4019-a29c-78089b23f519 kind: Event lastTimestamp: "2026-06-02T16:05:02Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-02T16:04:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:02Z" name: isvc-secondary-cffc11-predictor-85fd598f88-gbsnp.18b54f303ed573c2 namespace: kserve-ci-e2e-test resourceVersion: "18780" uid: 3fd3365f-6e39-4dff-9377-3f87ba5c32f7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T16:04:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-cffc11-predictor-85fd598f88-gbsnp namespace: kserve-ci-e2e-test resourceVersion: "18714" uid: 254dbfa2-2250-4019-a29c-78089b23f519 kind: Event lastTimestamp: "2026-06-02T16:05:02Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-02T16:04:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:02Z" name: isvc-secondary-cffc11-predictor-85fd598f88-gbsnp.18b54f303f551145 namespace: kserve-ci-e2e-test resourceVersion: "18781" uid: ebb2b79e-4c1f-4e46-8fc3-699933250c0f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-cffc11-predictor-85fd598f88-gbsnp namespace: kserve-ci-e2e-test resourceVersion: "18714" uid: 254dbfa2-2250-4019-a29c-78089b23f519 kind: Event lastTimestamp: "2026-06-02T16:05:05Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-cffc11-predictor-85fd598f88-gbsnp_kserve-ci-e2e-test(254dbfa2-2250-4019-a29c-78089b23f519) metadata: creationTimestamp: "2026-06-02T16:05:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:05Z" name: isvc-secondary-cffc11-predictor-85fd598f88-gbsnp.18b54f3263f13fa7 namespace: kserve-ci-e2e-test resourceVersion: "18809" uid: 45627e67-0c7b-4f9a-90ab-ee2c4d05a414 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:04:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-cffc11-predictor-85fd598f88 namespace: kserve-ci-e2e-test resourceVersion: "18702" uid: d28ffa17-1f01-48e1-a64b-6d70a64a59c5 kind: Event lastTimestamp: "2026-06-02T16:04:55Z" message: 'Created pod: isvc-secondary-cffc11-predictor-85fd598f88-gbsnp' metadata: creationTimestamp: "2026-06-02T16:04:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:04:55Z" name: isvc-secondary-cffc11-predictor-85fd598f88.18b54f301f35a839 namespace: kserve-ci-e2e-test resourceVersion: "18713" uid: 79293878-db6c-4bcc-9261-a9d8ff7b78d3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:04:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-cffc11-predictor namespace: kserve-ci-e2e-test resourceVersion: "18700" uid: 7a666692-9b6c-4a6e-94ae-d15efdf37606 kind: Event lastTimestamp: "2026-06-02T16:04:55Z" message: Scaled up replica set isvc-secondary-cffc11-predictor-85fd598f88 from 0 to 1 metadata: creationTimestamp: "2026-06-02T16:04:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:04:55Z" name: isvc-secondary-cffc11-predictor.18b54f301ba80d20 namespace: kserve-ci-e2e-test resourceVersion: "18704" uid: 4a036e04-9c51-4d32-9cc4-c116bfafbd0b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-cffc11-predictor namespace: kserve-ci-e2e-test resourceVersion: "18701" uid: 49800147-1d2c-4328-8819-884c6823b4d4 kind: Event lastTimestamp: "2026-06-02T16:05:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T16:05:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:05:10Z" name: isvc-secondary-cffc11-predictor.18b54f339a19c333 namespace: kserve-ci-e2e-test resourceVersion: "18854" uid: cd10806a-2027-493d-b9f7-c4caf976f038 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-cffc11-predictor namespace: kserve-ci-e2e-test resourceVersion: "18701" uid: 49800147-1d2c-4328-8819-884c6823b4d4 kind: Event lastTimestamp: "2026-06-02T16:05:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T16:05:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:05:10Z" name: isvc-secondary-cffc11-predictor.18b54f339a82f15a namespace: kserve-ci-e2e-test resourceVersion: "18855" uid: e502f136-db07-4bf9-a7d8-afe471c5cb21 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-06-02T16:04:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-cffc11 namespace: kserve-ci-e2e-test resourceVersion: "18690" uid: 3d1a8eeb-c910-4659-8610-8279ca9a8716 kind: Event lastTimestamp: "2026-06-02T16:05:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-02T16:04:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:05:06Z" name: isvc-secondary-cffc11.18b54f301b52ef01 namespace: kserve-ci-e2e-test resourceVersion: "18822" uid: c46ff388-235a-4018-95d8-d09c2ca017f3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:04:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-cffc11 namespace: kserve-ci-e2e-test resourceVersion: "18690" uid: 3d1a8eeb-c910-4659-8610-8279ca9a8716 kind: Event lastTimestamp: "2026-06-02T16:04:55Z" message: 'Failed to update status for InferenceService "isvc-secondary-cffc11": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-cffc11": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-02T16:04:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:04:55Z" name: isvc-secondary-cffc11.18b54f301be36f8d namespace: kserve-ci-e2e-test resourceVersion: "18706" uid: 54c74d4f-5409-4b29-a63e-18e6819f096d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:04:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-cffc11 namespace: kserve-ci-e2e-test resourceVersion: "18690" uid: 3d1a8eeb-c910-4659-8610-8279ca9a8716 kind: Event lastTimestamp: "2026-06-02T16:04:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-cffc11": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-02T16:04:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:04:55Z" name: isvc-secondary-cffc11.18b54f301be3c65c namespace: kserve-ci-e2e-test resourceVersion: "18707" uid: adaa2d2e-fb8c-4ca3-bc76-896686497f62 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-06-02T15:55:59.075935Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm namespace: kserve-ci-e2e-test resourceVersion: "13979" uid: 7fa53da0-d6c8-4d23-bfb8-cd87a7980cde kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm to ip-10-0-140-246.ec2.internal metadata: creationTimestamp: "2026-06-02T15:55:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-02T15:55:59Z" name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm.18b54eb3283aaea4 namespace: kserve-ci-e2e-test resourceVersion: "13990" uid: 71bdefc1-c47c-46ff-9e82-4bef969a359d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8ddcd7f6c-hs6d2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:55:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm namespace: kserve-ci-e2e-test resourceVersion: "13998" uid: 7fa53da0-d6c8-4d23-bfb8-cd87a7980cde kind: Event lastTimestamp: "2026-06-02T15:55:59Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-02T15:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-02T15:55:59Z" name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm.18b54eb341e42dda namespace: kserve-ci-e2e-test resourceVersion: "14030" uid: 5f041f26-2ebc-41e0-a989-5bc1a94eacec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:55:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: 7fa53da0-d6c8-4d23-bfb8-cd87a7980cde kind: Event lastTimestamp: "2026-06-02T15:55:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ca9e7e1d07975dffacc4b293d655f2a02df4615629916a1cdc022036c5a27a57" already present on machine metadata: creationTimestamp: "2026-06-02T15:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:55:59Z" name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm.18b54eb344028818 namespace: kserve-ci-e2e-test resourceVersion: "14032" uid: 12c5a797-6144-485e-b8c1-c39cc61f96b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:55:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: 7fa53da0-d6c8-4d23-bfb8-cd87a7980cde kind: Event lastTimestamp: "2026-06-02T15:55:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-02T15:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:55:59Z" name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm.18b54eb348896aee namespace: kserve-ci-e2e-test resourceVersion: "14035" uid: cc2aaa1f-71af-4d5a-805d-6c6f04dcfe2b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:55:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: 7fa53da0-d6c8-4d23-bfb8-cd87a7980cde kind: Event lastTimestamp: "2026-06-02T15:55:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-02T15:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:55:59Z" name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm.18b54eb348fec8a5 namespace: kserve-ci-e2e-test resourceVersion: "14037" uid: 822eeb9a-5c05-45c9-bb8d-282861964063 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: 7fa53da0-d6c8-4d23-bfb8-cd87a7980cde kind: Event lastTimestamp: "2026-06-02T15:56:03Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1560" already present on machine metadata: creationTimestamp: "2026-06-02T15:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:56:03Z" name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm.18b54eb443a72e5e namespace: kserve-ci-e2e-test resourceVersion: "14085" uid: 4467184c-8f99-4f40-856c-3d6d53fd77e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: 7fa53da0-d6c8-4d23-bfb8-cd87a7980cde kind: Event lastTimestamp: "2026-06-02T15:56:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-02T15:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:56:03Z" name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm.18b54eb4470f3567 namespace: kserve-ci-e2e-test resourceVersion: "14088" uid: 59c1c412-1a5c-4741-aa44-5f6da9a9deb2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: 7fa53da0-d6c8-4d23-bfb8-cd87a7980cde kind: Event lastTimestamp: "2026-06-02T15:56:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-02T15:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:56:03Z" name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm.18b54eb4477578bb namespace: kserve-ci-e2e-test resourceVersion: "14089" uid: 8eb52039-0ef7-4870-b51b-eee579b8b4df reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: 7fa53da0-d6c8-4d23-bfb8-cd87a7980cde kind: Event lastTimestamp: "2026-06-02T15:56:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-02T15:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:56:03Z" name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm.18b54eb447813fae namespace: kserve-ci-e2e-test resourceVersion: "14090" uid: 99c4ebf3-6bfd-4cde-a1de-dac194e5e942 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: 7fa53da0-d6c8-4d23-bfb8-cd87a7980cde kind: Event lastTimestamp: "2026-06-02T15:56:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-02T15:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:56:03Z" name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm.18b54eb44ac07bfb namespace: kserve-ci-e2e-test resourceVersion: "14093" uid: d6d03f0d-5751-42ba-ad5b-90522b47d870 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: 7fa53da0-d6c8-4d23-bfb8-cd87a7980cde kind: Event lastTimestamp: "2026-06-02T15:56:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T15:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:56:03Z" name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm.18b54eb44b2fec6f namespace: kserve-ci-e2e-test resourceVersion: "14094" uid: 0d9bd3a1-1ab2-48cd-93b9-8c05d0bbe5cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-06-02T15:56:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: 7fa53da0-d6c8-4d23-bfb8-cd87a7980cde kind: Event lastTimestamp: "2026-06-02T15:57:52Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-02T15:56:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:52Z" name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm.18b54eb4bb790869 namespace: kserve-ci-e2e-test resourceVersion: "15133" uid: a279612c-8470-41ed-991e-26fffce973b9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: 7fa53da0-d6c8-4d23-bfb8-cd87a7980cde kind: Event lastTimestamp: "2026-06-02T15:57:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-02T15:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:49Z" name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm.18b54eccee63b684 namespace: kserve-ci-e2e-test resourceVersion: "15013" uid: 973d0597-a5e3-4373-b5f9-a1b681bc3d8f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: 7fa53da0-d6c8-4d23-bfb8-cd87a7980cde kind: Event lastTimestamp: "2026-06-02T15:57:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T15:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:49Z" name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm.18b54eccee644418 namespace: kserve-ci-e2e-test resourceVersion: "15015" uid: 3831dfe2-0d9b-4dac-8a4f-1fba031cdc11 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: 7fa53da0-d6c8-4d23-bfb8-cd87a7980cde kind: Event lastTimestamp: "2026-06-02T15:57:51Z" message: 'Readiness probe failed: Get "https://10.134.0.20:8643/healthz": dial tcp 10.134.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-02T15:57:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:51Z" name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm.18b54ecd69bc631f namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 56a69698-cad5-4402-8861-45b4fac84855 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:55:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d namespace: kserve-ci-e2e-test resourceVersion: "13967" uid: 1a2d2d22-1b45-4ac0-81d1-63091c292433 kind: Event lastTimestamp: "2026-06-02T15:55:59Z" message: 'Created pod: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d-qbtzm' metadata: creationTimestamp: "2026-06-02T15:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:55:59Z" name: isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d.18b54eb327a5a3ec namespace: kserve-ci-e2e-test resourceVersion: "13986" uid: f8d7eeb9-69af-4f41-8d69-f56637be90d7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:55:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-a7b79-predictor namespace: kserve-ci-e2e-test resourceVersion: "13965" uid: ea062eee-37e5-4a5b-be97-0d3fc5507591 kind: Event lastTimestamp: "2026-06-02T15:55:58Z" message: Scaled up replica set isvc-sklearn-graph-raw-a7b79-predictor-7666ccb64d from 0 to 1 metadata: creationTimestamp: "2026-06-02T15:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:55:58Z" name: isvc-sklearn-graph-raw-a7b79-predictor.18b54eb322488d39 namespace: kserve-ci-e2e-test resourceVersion: "13969" uid: b6ff4358-4d64-4185-a57c-5b27bcb6d3f7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T15:56:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-a7b79-predictor namespace: kserve-ci-e2e-test resourceVersion: "13966" uid: 4739db3b-1ead-4b2a-bcc6-3e74dc6ab305 kind: Event lastTimestamp: "2026-06-02T15:56:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T15:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:56:29Z" name: isvc-sklearn-graph-raw-a7b79-predictor.18b54eb6a1f7b8a5 namespace: kserve-ci-e2e-test resourceVersion: "14302" uid: e97def00-e7bd-4e70-b443-932f4b41a4c4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T15:56:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-a7b79-predictor namespace: kserve-ci-e2e-test resourceVersion: "13966" uid: 4739db3b-1ead-4b2a-bcc6-3e74dc6ab305 kind: Event lastTimestamp: "2026-06-02T15:56:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T15:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:56:29Z" name: isvc-sklearn-graph-raw-a7b79-predictor.18b54eb6a2614045 namespace: kserve-ci-e2e-test resourceVersion: "14303" uid: 383731a3-2aad-4bb6-b770-5746058393e2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-02T15:56:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-a7b79-predictor namespace: kserve-ci-e2e-test resourceVersion: "14176" uid: 4739db3b-1ead-4b2a-bcc6-3e74dc6ab305 kind: Event lastTimestamp: "2026-06-02T15:57:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-02T15:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:57:44Z" name: isvc-sklearn-graph-raw-a7b79-predictor.18b54ebda1d1557b namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 1c20d73f-79c3-4a58-9874-c392b8bf2e4f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-06-02T15:56:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-a7b79-predictor namespace: kserve-ci-e2e-test resourceVersion: "14176" uid: 4739db3b-1ead-4b2a-bcc6-3e74dc6ab305 kind: Event lastTimestamp: "2026-06-02T15:57:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-02T15:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:57:44Z" name: isvc-sklearn-graph-raw-a7b79-predictor.18b54ebda26d0f07 namespace: kserve-ci-e2e-test resourceVersion: "14908" uid: 39204430-448d-4a56-94ee-d2d04f2a9b01 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-06-02T15:55:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-a7b79 namespace: kserve-ci-e2e-test resourceVersion: "13936" uid: 785d45b8-d5ba-48f9-9ae6-ef52760d809c kind: Event lastTimestamp: "2026-06-02T15:57:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-02T15:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:57:23Z" name: isvc-sklearn-graph-raw-a7b79.18b54eb3222d9b21 namespace: kserve-ci-e2e-test resourceVersion: "14711" uid: e5382d92-8612-4bbd-b714-42abd9445799 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:55:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-a7b79 namespace: kserve-ci-e2e-test resourceVersion: "13936" uid: 785d45b8-d5ba-48f9-9ae6-ef52760d809c kind: Event lastTimestamp: "2026-06-02T15:55:58Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-a7b79": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-a7b79": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-02T15:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:55:58Z" name: isvc-sklearn-graph-raw-a7b79.18b54eb322f8c9c9 namespace: kserve-ci-e2e-test resourceVersion: "13971" uid: 58df8afd-81d6-4527-8627-fe9ca21e2842 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-a7b79 namespace: kserve-ci-e2e-test resourceVersion: "14705" uid: 785d45b8-d5ba-48f9-9ae6-ef52760d809c kind: Event lastTimestamp: "2026-06-02T15:57:23Z" message: InferenceService [isvc-sklearn-graph-raw-a7b79] is Ready metadata: creationTimestamp: "2026-06-02T15:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:57:23Z" name: isvc-sklearn-graph-raw-a7b79.18b54ec6b68cf013 namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: c8eed7ee-1d4d-4c80-9e80-5c3cc5ee2833 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-02T15:57:49.963874Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx namespace: kserve-ci-e2e-test resourceVersion: "15030" uid: 5a5dc465-78aa-4712-940a-e7c7a6f27fff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx to ip-10-0-140-246.ec2.internal metadata: creationTimestamp: "2026-06-02T15:57:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-02T15:57:49Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx.18b54eccf9aa5ce2 namespace: kserve-ci-e2e-test resourceVersion: "15038" uid: 74e3c564-c130-4b92-b0fa-46041985d8fb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8ddcd7f6c-hs6d2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: 5a5dc465-78aa-4712-940a-e7c7a6f27fff kind: Event lastTimestamp: "2026-06-02T15:57:50Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-02T15:57:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-02T15:57:50Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx.18b54ecd13c320c5 namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: 3a452f07-705e-45f6-b291-d804f7c46099 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 5a5dc465-78aa-4712-940a-e7c7a6f27fff kind: Event lastTimestamp: "2026-06-02T15:57:50Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ca9e7e1d07975dffacc4b293d655f2a02df4615629916a1cdc022036c5a27a57" already present on machine metadata: creationTimestamp: "2026-06-02T15:57:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:50Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx.18b54ecd16ac0a76 namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: d9160b88-0a61-47fd-b216-6c929f05eef4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 5a5dc465-78aa-4712-940a-e7c7a6f27fff kind: Event lastTimestamp: "2026-06-02T15:57:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-02T15:57:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:50Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx.18b54ecd192c221e namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: a03d5971-c351-4684-8f38-87cdb67aa755 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 5a5dc465-78aa-4712-940a-e7c7a6f27fff kind: Event lastTimestamp: "2026-06-02T15:57:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-02T15:57:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:50Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx.18b54ecd199f5eea namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: d2c5cbba-4910-4597-98d1-dc06eeeb84be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 5a5dc465-78aa-4712-940a-e7c7a6f27fff kind: Event lastTimestamp: "2026-06-02T15:57:55Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1560" already present on machine metadata: creationTimestamp: "2026-06-02T15:57:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:55Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx.18b54ece31b8545a namespace: kserve-ci-e2e-test resourceVersion: "15150" uid: aa8946fd-1569-494d-b19c-797985ebc625 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 5a5dc465-78aa-4712-940a-e7c7a6f27fff kind: Event lastTimestamp: "2026-06-02T15:57:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-02T15:57:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:55Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx.18b54ece35a71fb3 namespace: kserve-ci-e2e-test resourceVersion: "15155" uid: 2c1b56a3-62a5-47cd-81b1-7d664bc77303 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 5a5dc465-78aa-4712-940a-e7c7a6f27fff kind: Event lastTimestamp: "2026-06-02T15:57:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-02T15:57:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:55Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx.18b54ece3638658e namespace: kserve-ci-e2e-test resourceVersion: "15164" uid: d5f3b3cf-62ed-4601-b435-a4b24387685f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 5a5dc465-78aa-4712-940a-e7c7a6f27fff kind: Event lastTimestamp: "2026-06-02T15:57:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-02T15:57:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:55Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx.18b54ece365f03c3 namespace: kserve-ci-e2e-test resourceVersion: "15166" uid: dc6456cd-7813-4b25-9cc2-d33b9cf05965 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 5a5dc465-78aa-4712-940a-e7c7a6f27fff kind: Event lastTimestamp: "2026-06-02T15:57:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-02T15:57:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:55Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx.18b54ece39d20e76 namespace: kserve-ci-e2e-test resourceVersion: "15169" uid: 5d653601-8604-4c6d-9261-91edfeefcc6d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 5a5dc465-78aa-4712-940a-e7c7a6f27fff kind: Event lastTimestamp: "2026-06-02T15:57:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T15:57:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:55Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx.18b54ece3a412fec namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: b79f4131-94f6-412c-8db9-93a56c9d44a0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-06-02T15:57:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 5a5dc465-78aa-4712-940a-e7c7a6f27fff kind: Event lastTimestamp: "2026-06-02T15:59:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-02T15:57:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:32Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx.18b54ece6dea99d3 namespace: kserve-ci-e2e-test resourceVersion: "16061" uid: 489511c5-6781-473f-a1f0-c8e78df261ca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 5a5dc465-78aa-4712-940a-e7c7a6f27fff kind: Event lastTimestamp: "2026-06-02T15:59:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-02T15:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:29Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx.18b54ee4427a964e namespace: kserve-ci-e2e-test resourceVersion: "15990" uid: 470d0548-d784-46fd-af85-59c39c0965c9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 5a5dc465-78aa-4712-940a-e7c7a6f27fff kind: Event lastTimestamp: "2026-06-02T15:59:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T15:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:29Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx.18b54ee4427b102a namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: 7a1f4883-7548-48aa-96d6-819dacb3f1e7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx namespace: kserve-ci-e2e-test resourceVersion: "15033" uid: 5a5dc465-78aa-4712-940a-e7c7a6f27fff kind: Event lastTimestamp: "2026-06-02T15:59:32Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-02T15:59:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:32Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx.18b54ee4c83d8602 namespace: kserve-ci-e2e-test resourceVersion: "16060" uid: d69a63c3-6b37-4575-ae70-9fdbc936d2b5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: 8635125b-cdc2-406c-9850-593c53428bce kind: Event lastTimestamp: "2026-06-02T15:57:49Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b-g9wrx' metadata: creationTimestamp: "2026-06-02T15:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:57:49Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b.18b54eccf8070a37 namespace: kserve-ci-e2e-test resourceVersion: "15032" uid: 98402db3-f1b4-46a2-92c9-fe9ef46c2135 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: 0bfccb89-5283-4be8-8d46-a65ccf5172f2 kind: Event lastTimestamp: "2026-06-02T15:57:49Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-ae9ae-predictor-6c79649c9b from 0 to 1 metadata: creationTimestamp: "2026-06-02T15:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:57:49Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor.18b54ecce944ea86 namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: 69145b0f-5b49-4ec9-b646-3892086dc6c7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T15:58:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: 1beae0cf-c26a-4606-821e-4607978e4fe2 kind: Event lastTimestamp: "2026-06-02T15:58:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T15:58:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:58:34Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor.18b54ed067749666 namespace: kserve-ci-e2e-test resourceVersion: "15480" uid: 85dc36cc-a19d-4493-8125-a5da9f9562bf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T15:58:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: 1beae0cf-c26a-4606-821e-4607978e4fe2 kind: Event lastTimestamp: "2026-06-02T15:58:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T15:58:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:58:34Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor.18b54ed06815a766 namespace: kserve-ci-e2e-test resourceVersion: "15482" uid: 70f2865f-7a68-4fe7-9ef2-1d7ef5fe1b8a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-02T15:58:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: 1beae0cf-c26a-4606-821e-4607978e4fe2 kind: Event lastTimestamp: "2026-06-02T15:59:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-02T15:58:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:59:19Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor.18b54ed3e7094956 namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: 2c5a790d-8e60-4094-b4c2-35f69e72b3f2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-02T15:58:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: 1beae0cf-c26a-4606-821e-4607978e4fe2 kind: Event lastTimestamp: "2026-06-02T15:59:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-02T15:58:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:59:19Z" name: isvc-sklearn-graph-raw-hpa-ae9ae-predictor.18b54ed3e788051f namespace: kserve-ci-e2e-test resourceVersion: "15853" uid: 79a0a2dc-4859-4a38-b81b-3ef3d3b309ab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-06-02T15:57:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-ae9ae namespace: kserve-ci-e2e-test resourceVersion: "14977" uid: ef4e1c31-4334-427f-9585-50e2fe2ef778 kind: Event lastTimestamp: "2026-06-02T15:59:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-02T15:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:59:02Z" name: isvc-sklearn-graph-raw-hpa-ae9ae.18b54ecce814d920 namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: b5df16b2-04bc-4b44-a06a-f5dc5d04103e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-ae9ae namespace: kserve-ci-e2e-test resourceVersion: "14977" uid: ef4e1c31-4334-427f-9585-50e2fe2ef778 kind: Event lastTimestamp: "2026-06-02T15:57:49Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-ae9ae": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-ae9ae": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-02T15:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:57:49Z" name: isvc-sklearn-graph-raw-hpa-ae9ae.18b54ecce9999ef1 namespace: kserve-ci-e2e-test resourceVersion: "15000" uid: 44995e6d-c22a-41c0-9021-39dde3a3e1eb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-ae9ae namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: ef4e1c31-4334-427f-9585-50e2fe2ef778 kind: Event lastTimestamp: "2026-06-02T15:59:02Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-ae9ae] is Ready metadata: creationTimestamp: "2026-06-02T15:59:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:59:02Z" name: isvc-sklearn-graph-raw-hpa-ae9ae.18b54eddd2db5927 namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: c37651d9-475d-4a9c-ad6e-ab108aeed5ed reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-02T16:01:15.307150Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6 namespace: kserve-ci-e2e-test resourceVersion: "16947" uid: 0061fcda-676e-40ef-a3af-7c90f1ea2a65 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6 to ip-10-0-140-246.ec2.internal metadata: creationTimestamp: "2026-06-02T16:01:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-02T16:01:15Z" name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6.18b54efcc9140fa9 namespace: kserve-ci-e2e-test resourceVersion: "16951" uid: 9928a8a6-4d61-47cf-9c4b-885a834d4e4a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8ddcd7f6c-hs6d2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:15Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6 namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 0061fcda-676e-40ef-a3af-7c90f1ea2a65 kind: Event lastTimestamp: "2026-06-02T16:01:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-261e5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-02T16:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:01:15Z" name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6.18b54efcd3d9c259 namespace: kserve-ci-e2e-test resourceVersion: "16974" uid: 635af606-c2f9-4203-be4c-7420711ebf8e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6 namespace: kserve-ci-e2e-test resourceVersion: "16962" uid: 0061fcda-676e-40ef-a3af-7c90f1ea2a65 kind: Event lastTimestamp: "2026-06-02T16:01:16Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-02T16:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-02T16:01:16Z" name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6.18b54efd0626d77a namespace: kserve-ci-e2e-test resourceVersion: "16993" uid: 0f8d3a96-cc66-493f-b700-3a52f73e8bdb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6 namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 0061fcda-676e-40ef-a3af-7c90f1ea2a65 kind: Event lastTimestamp: "2026-06-02T16:01:16Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ca9e7e1d07975dffacc4b293d655f2a02df4615629916a1cdc022036c5a27a57" already present on machine metadata: creationTimestamp: "2026-06-02T16:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:01:16Z" name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6.18b54efd07c75989 namespace: kserve-ci-e2e-test resourceVersion: "16995" uid: a0e0ea89-f3cb-417f-a3e7-a8c67542e5f8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6 namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 0061fcda-676e-40ef-a3af-7c90f1ea2a65 kind: Event lastTimestamp: "2026-06-02T16:01:16Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-02T16:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:01:16Z" name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6.18b54efd0b4a6591 namespace: kserve-ci-e2e-test resourceVersion: "16997" uid: 161a1860-848c-4cfa-8c80-a1c7aa5c400b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6 namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 0061fcda-676e-40ef-a3af-7c90f1ea2a65 kind: Event lastTimestamp: "2026-06-02T16:01:16Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-02T16:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:01:16Z" name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6.18b54efd0ba8c30e namespace: kserve-ci-e2e-test resourceVersion: "16998" uid: f7aefdbb-ddfe-4b54-b5bb-ac49ab96aa1d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6 namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 0061fcda-676e-40ef-a3af-7c90f1ea2a65 kind: Event lastTimestamp: "2026-06-02T16:01:20Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1560" already present on machine metadata: creationTimestamp: "2026-06-02T16:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:01:20Z" name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6.18b54efe131a36a1 namespace: kserve-ci-e2e-test resourceVersion: "17036" uid: ad2959ba-3c5e-4acf-b5e7-1480ddece3fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6 namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 0061fcda-676e-40ef-a3af-7c90f1ea2a65 kind: Event lastTimestamp: "2026-06-02T16:01:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-02T16:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:01:20Z" name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6.18b54efe16255133 namespace: kserve-ci-e2e-test resourceVersion: "17038" uid: 983bf735-5ab3-48d8-90e1-ec3872d5a424 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6 namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 0061fcda-676e-40ef-a3af-7c90f1ea2a65 kind: Event lastTimestamp: "2026-06-02T16:01:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-02T16:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:01:20Z" name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6.18b54efe16852a88 namespace: kserve-ci-e2e-test resourceVersion: "17039" uid: 2617fc99-5840-4de3-9308-d1b089e49c66 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6 namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 0061fcda-676e-40ef-a3af-7c90f1ea2a65 kind: Event lastTimestamp: "2026-06-02T16:01:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-02T16:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:01:20Z" name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6.18b54efe1690f0a5 namespace: kserve-ci-e2e-test resourceVersion: "17040" uid: a0a3421d-61f3-44f5-b196-dab75b0a8e9e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6 namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 0061fcda-676e-40ef-a3af-7c90f1ea2a65 kind: Event lastTimestamp: "2026-06-02T16:01:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-02T16:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:01:20Z" name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6.18b54efe198b64bb namespace: kserve-ci-e2e-test resourceVersion: "17044" uid: 0320490f-94a3-4599-9c9c-f0773c7fd571 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6 namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 0061fcda-676e-40ef-a3af-7c90f1ea2a65 kind: Event lastTimestamp: "2026-06-02T16:01:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T16:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:01:20Z" name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6.18b54efe19e8a076 namespace: kserve-ci-e2e-test resourceVersion: "17045" uid: c052f56e-810c-4d67-8d35-09ef15474278 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 15 eventTime: null firstTimestamp: "2026-06-02T16:01:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6 namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 0061fcda-676e-40ef-a3af-7c90f1ea2a65 kind: Event lastTimestamp: "2026-06-02T16:03:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-02T16:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:03:24Z" name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6.18b54efe8ac96cf2 namespace: kserve-ci-e2e-test resourceVersion: "17934" uid: 11be9318-2e06-483f-9c7e-5cd9aabc47f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:03:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6 namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 0061fcda-676e-40ef-a3af-7c90f1ea2a65 kind: Event lastTimestamp: "2026-06-02T16:03:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-02T16:03:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:03:35Z" name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6.18b54f1d77e9569d namespace: kserve-ci-e2e-test resourceVersion: "18053" uid: 25d564ff-e14f-4d1b-81c1-e5fddbaf35cb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:03:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6 namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 0061fcda-676e-40ef-a3af-7c90f1ea2a65 kind: Event lastTimestamp: "2026-06-02T16:03:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T16:03:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:03:35Z" name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6.18b54f1d77e9f23b namespace: kserve-ci-e2e-test resourceVersion: "18054" uid: 7e2af156-dd19-4ee9-9f91-8b5e47b1f37b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T16:03:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6 namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 0061fcda-676e-40ef-a3af-7c90f1ea2a65 kind: Event lastTimestamp: "2026-06-02T16:03:43Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-02T16:03:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:03:43Z" name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6.18b54f1e354deabd namespace: kserve-ci-e2e-test resourceVersion: "18160" uid: f815d4cc-428e-44ab-aaf7-f4e286192972 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598 namespace: kserve-ci-e2e-test resourceVersion: "16927" uid: c4aaf811-bcee-4570-a3b8-9321876c0485 kind: Event lastTimestamp: "2026-06-02T16:01:15Z" message: 'Created pod: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598-b9gc6' metadata: creationTimestamp: "2026-06-02T16:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:01:15Z" name: isvc-sklearn-scale-raw-261e5-predictor-666fd5f598.18b54efcc8cb4e3b namespace: kserve-ci-e2e-test resourceVersion: "16949" uid: a6f426b5-d11c-48e1-a8d3-6a8e40eb450f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-261e5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16921" uid: 20e6c5ed-4456-4bf3-9295-db8e582b9332 kind: Event lastTimestamp: "2026-06-02T16:01:15Z" message: Scaled up replica set isvc-sklearn-scale-raw-261e5-predictor-666fd5f598 from 0 to 1 metadata: creationTimestamp: "2026-06-02T16:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:01:15Z" name: isvc-sklearn-scale-raw-261e5-predictor.18b54efcc476653a namespace: kserve-ci-e2e-test resourceVersion: "16932" uid: e543d0f5-11e2-4697-aa01-eaf78cdaca4b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T16:01:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-261e5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: 9a17bdb6-ce3c-4e17-b56c-cd42d0a995a4 kind: Event lastTimestamp: "2026-06-02T16:01:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T16:01:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:01:45Z" name: isvc-sklearn-scale-raw-261e5-predictor.18b54f004360e6af namespace: kserve-ci-e2e-test resourceVersion: "17230" uid: 7fb011e9-2294-494d-9a6f-ebdcecba04dd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T16:01:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-261e5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: 9a17bdb6-ce3c-4e17-b56c-cd42d0a995a4 kind: Event lastTimestamp: "2026-06-02T16:01:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T16:01:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:01:45Z" name: isvc-sklearn-scale-raw-261e5-predictor.18b54f0043d27ca1 namespace: kserve-ci-e2e-test resourceVersion: "17231" uid: 7f0c74d5-db6c-4b4c-a781-c8a2e294c4ea reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-06-02T16:02:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-261e5-predictor namespace: kserve-ci-e2e-test resourceVersion: "17118" uid: 9a17bdb6-ce3c-4e17-b56c-cd42d0a995a4 kind: Event lastTimestamp: "2026-06-02T16:03:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-02T16:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:03:30Z" name: isvc-sklearn-scale-raw-261e5-predictor.18b54f07421af5ec namespace: kserve-ci-e2e-test resourceVersion: "17975" uid: a75bd7cf-77e9-46b9-91f8-aac8235f5fbf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-06-02T16:02:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-261e5-predictor namespace: kserve-ci-e2e-test resourceVersion: "17118" uid: 9a17bdb6-ce3c-4e17-b56c-cd42d0a995a4 kind: Event lastTimestamp: "2026-06-02T16:03:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-02T16:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:03:30Z" name: isvc-sklearn-scale-raw-261e5-predictor.18b54f07428972b6 namespace: kserve-ci-e2e-test resourceVersion: "17977" uid: 3be8baa3-d994-4952-8e3d-83a1734e8709 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-06-02T16:01:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-261e5 namespace: kserve-ci-e2e-test resourceVersion: "16905" uid: f3426d2d-484d-41df-897a-f4a887ca0e60 kind: Event lastTimestamp: "2026-06-02T16:03:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-02T16:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:03:35Z" name: isvc-sklearn-scale-raw-261e5.18b54efcc4815a80 namespace: kserve-ci-e2e-test resourceVersion: "18019" uid: 8f3ea02a-2aa1-483f-b40b-2181851f78f0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-261e5 namespace: kserve-ci-e2e-test resourceVersion: "16905" uid: f3426d2d-484d-41df-897a-f4a887ca0e60 kind: Event lastTimestamp: "2026-06-02T16:01:15Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-261e5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-261e5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-02T16:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:01:15Z" name: isvc-sklearn-scale-raw-261e5.18b54efcc50bd1f8 namespace: kserve-ci-e2e-test resourceVersion: "16937" uid: 56ec9f33-9c20-4413-851e-da42b6cd1153 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:03:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-261e5 namespace: kserve-ci-e2e-test resourceVersion: "18017" uid: f3426d2d-484d-41df-897a-f4a887ca0e60 kind: Event lastTimestamp: "2026-06-02T16:03:35Z" message: InferenceService [isvc-sklearn-scale-raw-261e5] is Ready metadata: creationTimestamp: "2026-06-02T16:03:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:03:35Z" name: isvc-sklearn-scale-raw-261e5.18b54f1d4fe4bc31 namespace: kserve-ci-e2e-test resourceVersion: "18018" uid: 0080c1e5-e46e-4fa1-adab-d33837870518 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-02T15:55:59.185689Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5 namespace: kserve-ci-e2e-test resourceVersion: "14008" uid: 5df37958-2286-4d27-898c-9628699e3ff7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5 to ip-10-0-140-246.ec2.internal metadata: creationTimestamp: "2026-06-02T15:55:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-02T15:55:59Z" name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5.18b54eb32ec56fa7 namespace: kserve-ci-e2e-test resourceVersion: "14015" uid: 47333f15-5e35-4e87-ad32-da2b7fa1ad69 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8ddcd7f6c-hs6d2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:55:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5 namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: 5df37958-2286-4d27-898c-9628699e3ff7 kind: Event lastTimestamp: "2026-06-02T15:55:59Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-02T15:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-02T15:55:59Z" name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5.18b54eb348445851 namespace: kserve-ci-e2e-test resourceVersion: "14034" uid: eded4a10-cd57-457b-8897-fbb14db34c9d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:55:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5 namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: 5df37958-2286-4d27-898c-9628699e3ff7 kind: Event lastTimestamp: "2026-06-02T15:55:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ca9e7e1d07975dffacc4b293d655f2a02df4615629916a1cdc022036c5a27a57" already present on machine metadata: creationTimestamp: "2026-06-02T15:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:55:59Z" name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5.18b54eb349cecbdb namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: 6e39c869-4c6b-42f2-8590-60064b5b775d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:55:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5 namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: 5df37958-2286-4d27-898c-9628699e3ff7 kind: Event lastTimestamp: "2026-06-02T15:55:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-02T15:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:55:59Z" name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5.18b54eb34c5192e6 namespace: kserve-ci-e2e-test resourceVersion: "14040" uid: 03727861-15c5-4ca1-afb8-9d2b575d0940 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:55:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5 namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: 5df37958-2286-4d27-898c-9628699e3ff7 kind: Event lastTimestamp: "2026-06-02T15:55:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-02T15:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:55:59Z" name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5.18b54eb34cb499a4 namespace: kserve-ci-e2e-test resourceVersion: "14041" uid: 95dce780-3d52-49ca-9521-091ee71f9804 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:56:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5 namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: 5df37958-2286-4d27-898c-9628699e3ff7 kind: Event lastTimestamp: "2026-06-02T15:56:03Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-06-02T15:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:56:03Z" name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5.18b54eb443998e2e namespace: kserve-ci-e2e-test resourceVersion: "14084" uid: 3069d250-91cf-40e5-9550-fe3693ddd1d3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:56:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5 namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: 5df37958-2286-4d27-898c-9628699e3ff7 kind: Event lastTimestamp: "2026-06-02T15:56:20Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 16.379s (16.379s including waiting). Image size: 1306325243 bytes.' metadata: creationTimestamp: "2026-06-02T15:56:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:56:20Z" name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5.18b54eb813e915cc namespace: kserve-ci-e2e-test resourceVersion: "14223" uid: b5b8844a-c152-4591-8ca7-6ccefbb86e0c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:56:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5 namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: 5df37958-2286-4d27-898c-9628699e3ff7 kind: Event lastTimestamp: "2026-06-02T15:56:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-02T15:56:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:56:20Z" name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5.18b54eb8180cea60 namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: 3d554077-9881-493b-a2d2-e83a4bf64bd6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:56:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5 namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: 5df37958-2286-4d27-898c-9628699e3ff7 kind: Event lastTimestamp: "2026-06-02T15:56:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-02T15:56:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:56:20Z" name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5.18b54eb818ab89e8 namespace: kserve-ci-e2e-test resourceVersion: "14225" uid: cad42518-f465-491a-9025-23750142d07a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:56:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5 namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: 5df37958-2286-4d27-898c-9628699e3ff7 kind: Event lastTimestamp: "2026-06-02T15:56:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-02T15:56:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:56:20Z" name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5.18b54eb818b8f98f namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: 919257c9-add7-4f87-b1bb-21764bd02668 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:56:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5 namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: 5df37958-2286-4d27-898c-9628699e3ff7 kind: Event lastTimestamp: "2026-06-02T15:56:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-02T15:56:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:56:20Z" name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5.18b54eb81cb75fd1 namespace: kserve-ci-e2e-test resourceVersion: "14228" uid: a85dc304-8de2-4854-8943-12095b492ef2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:56:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5 namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: 5df37958-2286-4d27-898c-9628699e3ff7 kind: Event lastTimestamp: "2026-06-02T15:56:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T15:56:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:56:20Z" name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5.18b54eb81d62acf7 namespace: kserve-ci-e2e-test resourceVersion: "14230" uid: f947c510-5dde-4ced-8b2d-9a90ab849115 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-06-02T15:56:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5 namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: 5df37958-2286-4d27-898c-9628699e3ff7 kind: Event lastTimestamp: "2026-06-02T15:57:17Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-02T15:56:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:17Z" name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5.18b54eb87876d840 namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: b9672b3c-fc32-4143-ae2b-b0691893ee89 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5 namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: 5df37958-2286-4d27-898c-9628699e3ff7 kind: Event lastTimestamp: "2026-06-02T15:57:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-02T15:57:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:50Z" name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5.18b54ecd0eb4fef4 namespace: kserve-ci-e2e-test resourceVersion: "15084" uid: 1c03b110-f43d-454d-bd7c-1cf60e48a86e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5 namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: 5df37958-2286-4d27-898c-9628699e3ff7 kind: Event lastTimestamp: "2026-06-02T15:57:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T15:57:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:50Z" name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5.18b54ecd0eb56cb7 namespace: kserve-ci-e2e-test resourceVersion: "15085" uid: 83024bff-7636-4640-92c2-410d21c444cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5 namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: 5df37958-2286-4d27-898c-9628699e3ff7 kind: Event lastTimestamp: "2026-06-02T15:57:52Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-02T15:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:52Z" name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5.18b54ecda8af4251 namespace: kserve-ci-e2e-test resourceVersion: "15132" uid: d6f50c78-4519-4828-86ad-04a026dae1d0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:55:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85 namespace: kserve-ci-e2e-test resourceVersion: "13997" uid: 999fbb43-8af7-457b-9dae-523c5033371a kind: Event lastTimestamp: "2026-06-02T15:55:59Z" message: 'Created pod: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85-mmkz5' metadata: creationTimestamp: "2026-06-02T15:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:55:59Z" name: isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85.18b54eb32e0d62b8 namespace: kserve-ci-e2e-test resourceVersion: "14010" uid: 9c7ba5b4-42bf-406b-87eb-76ada965e63c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:55:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-a7b79-predictor namespace: kserve-ci-e2e-test resourceVersion: "13994" uid: 4ac55f28-a440-48ed-9e72-8854b2117613 kind: Event lastTimestamp: "2026-06-02T15:55:59Z" message: Scaled up replica set isvc-xgboost-graph-raw-a7b79-predictor-79b5ccd85 from 0 to 1 metadata: creationTimestamp: "2026-06-02T15:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:55:59Z" name: isvc-xgboost-graph-raw-a7b79-predictor.18b54eb32a51aa06 namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 33db5b5e-be41-414f-9861-30d8bcef6367 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-02T15:56:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-a7b79-predictor namespace: kserve-ci-e2e-test resourceVersion: "13996" uid: d73fe4eb-1cc3-40bd-8e3d-b74306a01513 kind: Event lastTimestamp: "2026-06-02T15:56:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T15:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:56:44Z" name: isvc-xgboost-graph-raw-a7b79-predictor.18b54eb6a9185f5b namespace: kserve-ci-e2e-test resourceVersion: "14414" uid: a768a83a-c253-4774-85a1-13198f922f56 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-02T15:56:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-a7b79-predictor namespace: kserve-ci-e2e-test resourceVersion: "13996" uid: d73fe4eb-1cc3-40bd-8e3d-b74306a01513 kind: Event lastTimestamp: "2026-06-02T15:56:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T15:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:56:44Z" name: isvc-xgboost-graph-raw-a7b79-predictor.18b54eb6a99f8bb3 namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: db369135-ac59-454d-9c4d-b7b3481409ab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-02T15:56:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-a7b79-predictor namespace: kserve-ci-e2e-test resourceVersion: "14180" uid: d73fe4eb-1cc3-40bd-8e3d-b74306a01513 kind: Event lastTimestamp: "2026-06-02T15:57:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-02T15:56:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:57:44Z" name: isvc-xgboost-graph-raw-a7b79-predictor.18b54ec126d0a6bc namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: b596d686-8084-4e53-81c2-dda299d9ae6f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-06-02T15:56:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-a7b79-predictor namespace: kserve-ci-e2e-test resourceVersion: "14180" uid: d73fe4eb-1cc3-40bd-8e3d-b74306a01513 kind: Event lastTimestamp: "2026-06-02T15:57:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-02T15:56:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:57:44Z" name: isvc-xgboost-graph-raw-a7b79-predictor.18b54ec127b853fc namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: 3c27c535-88eb-4cc1-ad25-d2a13a79c972 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-06-02T15:55:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-a7b79 namespace: kserve-ci-e2e-test resourceVersion: "13976" uid: d6489e61-9330-4979-bce5-ce9cb9967d25 kind: Event lastTimestamp: "2026-06-02T15:57:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-02T15:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:57:28Z" name: isvc-xgboost-graph-raw-a7b79.18b54eb32a557b5d namespace: kserve-ci-e2e-test resourceVersion: "14755" uid: b8ca8489-ea1d-4831-9265-5153457bc5fe reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-a7b79 namespace: kserve-ci-e2e-test resourceVersion: "14752" uid: d6489e61-9330-4979-bce5-ce9cb9967d25 kind: Event lastTimestamp: "2026-06-02T15:57:28Z" message: InferenceService [isvc-xgboost-graph-raw-a7b79] is Ready metadata: creationTimestamp: "2026-06-02T15:57:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:57:28Z" name: isvc-xgboost-graph-raw-a7b79.18b54ec7e33f8860 namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: ff20ab6d-a5cc-43b7-93ca-1996f316afc0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-02T15:57:50.105927Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 52548776-159b-4aa2-aa31-33accbfffe9f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww to ip-10-0-140-246.ec2.internal metadata: creationTimestamp: "2026-06-02T15:57:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-02T15:57:50Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww.18b54ecd0221e799 namespace: kserve-ci-e2e-test resourceVersion: "15058" uid: debbb3c1-93c2-4743-bb7b-8a393599ef91 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8ddcd7f6c-hs6d2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: 52548776-159b-4aa2-aa31-33accbfffe9f kind: Event lastTimestamp: "2026-06-02T15:57:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-hpa-ae9ae-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-06-02T15:57:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:50Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww.18b54ecd0ab03b96 namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 4bdef673-cf72-4b3f-9015-88f561be8774 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww namespace: kserve-ci-e2e-test resourceVersion: "15060" uid: 52548776-159b-4aa2-aa31-33accbfffe9f kind: Event lastTimestamp: "2026-06-02T15:57:51Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-02T15:57:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-02T15:57:51Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww.18b54ecd3f03c411 namespace: kserve-ci-e2e-test resourceVersion: "15097" uid: 87fea170-b817-4e51-98e5-71a0fbb0c964 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: 52548776-159b-4aa2-aa31-33accbfffe9f kind: Event lastTimestamp: "2026-06-02T15:57:51Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ca9e7e1d07975dffacc4b293d655f2a02df4615629916a1cdc022036c5a27a57" already present on machine metadata: creationTimestamp: "2026-06-02T15:57:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:51Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww.18b54ecd41ea77e5 namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: 2f9d91fd-1b21-4c79-9364-1cd7462baccd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: 52548776-159b-4aa2-aa31-33accbfffe9f kind: Event lastTimestamp: "2026-06-02T15:57:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-02T15:57:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:51Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww.18b54ecd464a6000 namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: 2b13477e-0a31-4c01-a172-70dd7dd09fba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: 52548776-159b-4aa2-aa31-33accbfffe9f kind: Event lastTimestamp: "2026-06-02T15:57:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-02T15:57:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:51Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww.18b54ecd46b7417d namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 2fcea8be-a4d4-4085-823f-d470523addf6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: 52548776-159b-4aa2-aa31-33accbfffe9f kind: Event lastTimestamp: "2026-06-02T15:57:55Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-06-02T15:57:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:55Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww.18b54ece31659f04 namespace: kserve-ci-e2e-test resourceVersion: "15149" uid: 9e6d04d5-e264-48eb-90d6-c67a6359c7c3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: 52548776-159b-4aa2-aa31-33accbfffe9f kind: Event lastTimestamp: "2026-06-02T15:57:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-02T15:57:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:55Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww.18b54ece35a7ec6d namespace: kserve-ci-e2e-test resourceVersion: "15156" uid: f7cca5dc-5cfc-4cb3-bc1c-7057e8eee386 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: 52548776-159b-4aa2-aa31-33accbfffe9f kind: Event lastTimestamp: "2026-06-02T15:57:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-02T15:57:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:55Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww.18b54ece36333e98 namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 79665b1b-126d-441a-b7c7-828885a8304d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: 52548776-159b-4aa2-aa31-33accbfffe9f kind: Event lastTimestamp: "2026-06-02T15:57:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-02T15:57:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:55Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww.18b54ece36602380 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: b4da27b1-87b5-4d1d-8539-3f86ee50ff44 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: 52548776-159b-4aa2-aa31-33accbfffe9f kind: Event lastTimestamp: "2026-06-02T15:57:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-02T15:57:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:55Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww.18b54ece39c9ba6b namespace: kserve-ci-e2e-test resourceVersion: "15168" uid: 8efdd782-e7f1-4060-a50f-9fe485e58167 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: 52548776-159b-4aa2-aa31-33accbfffe9f kind: Event lastTimestamp: "2026-06-02T15:57:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T15:57:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:55Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww.18b54ece3a3ad8fa namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: c9d2b7ae-e1f1-4a7b-a009-b084b5d9dad9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-06-02T15:57:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: 52548776-159b-4aa2-aa31-33accbfffe9f kind: Event lastTimestamp: "2026-06-02T15:59:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-02T15:57:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:33Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww.18b54ecea9cfbb3b namespace: kserve-ci-e2e-test resourceVersion: "16072" uid: a47d102a-ed8e-477d-a4f5-d1b96058cebf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: 52548776-159b-4aa2-aa31-33accbfffe9f kind: Event lastTimestamp: "2026-06-02T15:59:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-02T15:59:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:30Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww.18b54ee44b211581 namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 761ca38f-951c-48fd-89b6-bc580e6047ca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: 52548776-159b-4aa2-aa31-33accbfffe9f kind: Event lastTimestamp: "2026-06-02T15:59:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T15:59:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:30Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww.18b54ee44b219045 namespace: kserve-ci-e2e-test resourceVersion: "16027" uid: 7d4a26e2-432b-4dc0-ba60-9755d74db1f6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: 52548776-159b-4aa2-aa31-33accbfffe9f kind: Event lastTimestamp: "2026-06-02T15:59:33Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-02T15:59:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:33Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww.18b54ee503f419e5 namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: 19f0f016-ae7b-4991-ad4c-8f2c025dbac4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7 namespace: kserve-ci-e2e-test resourceVersion: "15025" uid: c3ce3149-124b-4f71-bf5f-5afad5603e8e kind: Event lastTimestamp: "2026-06-02T15:57:50Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7-fk8ww' metadata: creationTimestamp: "2026-06-02T15:57:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:57:50Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7.18b54eccffe96ce0 namespace: kserve-ci-e2e-test resourceVersion: "15050" uid: ab9c529f-5135-496d-84ee-adccd38093d7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: a1c2167d-648a-433d-988c-415c10eba61b kind: Event lastTimestamp: "2026-06-02T15:57:49Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-ae9ae-predictor-5bf88f9fc7 from 0 to 1 metadata: creationTimestamp: "2026-06-02T15:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:57:49Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor.18b54eccf60d06ae namespace: kserve-ci-e2e-test resourceVersion: "15028" uid: 6a602bc4-ad16-4bfc-9cb5-f25444caf38e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-02T15:58:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "15027" uid: 22c789a9-cbe2-4b7c-8448-6a2bfae5af01 kind: Event lastTimestamp: "2026-06-02T15:58:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T15:58:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:58:34Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor.18b54ed076157f92 namespace: kserve-ci-e2e-test resourceVersion: "15483" uid: 283d2da8-21c3-4394-8ce6-6d78f3711d61 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-02T15:58:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "15027" uid: 22c789a9-cbe2-4b7c-8448-6a2bfae5af01 kind: Event lastTimestamp: "2026-06-02T15:58:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T15:58:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:58:34Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor.18b54ed076796d2a namespace: kserve-ci-e2e-test resourceVersion: "15484" uid: f27a7edc-25ce-42da-ab19-3fee79da3a2b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-02T15:58:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: 22c789a9-cbe2-4b7c-8448-6a2bfae5af01 kind: Event lastTimestamp: "2026-06-02T15:59:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-02T15:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:59:20Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor.18b54edaf378c125 namespace: kserve-ci-e2e-test resourceVersion: "15855" uid: 51895cfa-12de-46f8-a253-e3e71960776d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-02T15:58:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: 22c789a9-cbe2-4b7c-8448-6a2bfae5af01 kind: Event lastTimestamp: "2026-06-02T15:59:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-02T15:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:59:20Z" name: isvc-xgboost-graph-raw-hpa-ae9ae-predictor.18b54edaf412c36b namespace: kserve-ci-e2e-test resourceVersion: "15856" uid: 327535dd-f2e2-48f5-b19a-7c0957b574b7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-06-02T15:57:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-ae9ae namespace: kserve-ci-e2e-test resourceVersion: "15008" uid: a3f23945-d9a7-4ea8-86d2-90b3aeb3170e kind: Event lastTimestamp: "2026-06-02T15:59:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-02T15:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:59:03Z" name: isvc-xgboost-graph-raw-hpa-ae9ae.18b54eccf6ec4466 namespace: kserve-ci-e2e-test resourceVersion: "15705" uid: 24c1c666-e03c-4102-a48a-b82733e46e4d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-ae9ae namespace: kserve-ci-e2e-test resourceVersion: "15008" uid: a3f23945-d9a7-4ea8-86d2-90b3aeb3170e kind: Event lastTimestamp: "2026-06-02T15:57:49Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-ae9ae": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-ae9ae": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-02T15:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:57:49Z" name: isvc-xgboost-graph-raw-hpa-ae9ae.18b54eccf85b3e0e namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 0dd47709-b9b9-4940-a644-4c3ef1d00541 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-ae9ae namespace: kserve-ci-e2e-test resourceVersion: "15703" uid: a3f23945-d9a7-4ea8-86d2-90b3aeb3170e kind: Event lastTimestamp: "2026-06-02T15:59:03Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-ae9ae] is Ready metadata: creationTimestamp: "2026-06-02T15:59:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:59:03Z" name: isvc-xgboost-graph-raw-hpa-ae9ae.18b54ede0e0aecf3 namespace: kserve-ci-e2e-test resourceVersion: "15704" uid: eca709b0-f637-4fe5-98a0-def49e2b8ce9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-02T15:59:30.011678Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2 namespace: kserve-ci-e2e-test resourceVersion: "15995" uid: 14a65876-5613-49f5-89ad-6988b8d0aee7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-0d859-predictor-58bd984d44-v47p2 to ip-10-0-140-246.ec2.internal metadata: creationTimestamp: "2026-06-02T15:59:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-02T15:59:30Z" name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2.18b54ee444fabb0d namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: 71d225eb-b81f-4adf-a5e5-1051185223d6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8ddcd7f6c-hs6d2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:30Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2 namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: 14a65876-5613-49f5-89ad-6988b8d0aee7 kind: Event lastTimestamp: "2026-06-02T15:59:30Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-02T15:59:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-02T15:59:30Z" name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2.18b54ee45e182fae namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: a0648457-9b03-4eed-b3a8-02183f2461ae reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2 namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 14a65876-5613-49f5-89ad-6988b8d0aee7 kind: Event lastTimestamp: "2026-06-02T15:59:30Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-06-02T15:59:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:30Z" name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2.18b54ee45ff62891 namespace: kserve-ci-e2e-test resourceVersion: "16037" uid: 0e4ea524-c009-49ac-9844-267cf822de61 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2 namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 14a65876-5613-49f5-89ad-6988b8d0aee7 kind: Event lastTimestamp: "2026-06-02T15:59:31Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.063s (1.063s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-06-02T15:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:31Z" name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2.18b54ee49f5325d3 namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: af05fbe2-c526-4f10-99e0-9d52196695dc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2 namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 14a65876-5613-49f5-89ad-6988b8d0aee7 kind: Event lastTimestamp: "2026-06-02T15:59:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-02T15:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:31Z" name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2.18b54ee4a2e7dfaa namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 610a2c22-9809-4109-bbbd-eeddb2d709f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2 namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 14a65876-5613-49f5-89ad-6988b8d0aee7 kind: Event lastTimestamp: "2026-06-02T15:59:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-02T15:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:31Z" name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2.18b54ee4a36dd886 namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: b2083a81-51c2-47b3-aa0a-bbdd3384a076 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2 namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 14a65876-5613-49f5-89ad-6988b8d0aee7 kind: Event lastTimestamp: "2026-06-02T15:59:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-02T15:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:31Z" name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2.18b54ee4a3793cf9 namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: 5b24193d-948b-4bed-b63b-5376092768fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2 namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 14a65876-5613-49f5-89ad-6988b8d0aee7 kind: Event lastTimestamp: "2026-06-02T15:59:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-02T15:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:31Z" name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2.18b54ee4a616b0e5 namespace: kserve-ci-e2e-test resourceVersion: "16050" uid: 5f9ad3ab-727d-49ed-819e-5eb619b5692d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2 namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 14a65876-5613-49f5-89ad-6988b8d0aee7 kind: Event lastTimestamp: "2026-06-02T15:59:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T15:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:31Z" name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2.18b54ee4a68c137c namespace: kserve-ci-e2e-test resourceVersion: "16051" uid: ebaa1d7e-e6ce-4ffa-9f09-b54f9145a790 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2 namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 14a65876-5613-49f5-89ad-6988b8d0aee7 kind: Event lastTimestamp: "2026-06-02T16:01:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-02T16:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:01:15Z" name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2.18b54efccc9ba258 namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: b69f036e-5548-4dd6-8fa3-40c949c933b6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2 namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 14a65876-5613-49f5-89ad-6988b8d0aee7 kind: Event lastTimestamp: "2026-06-02T16:01:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T16:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:01:15Z" name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2.18b54efccc9bfe68 namespace: kserve-ci-e2e-test resourceVersion: "16973" uid: 13b6e78b-0d9b-4d25-b1d7-6ccd0acccbe7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:01:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2 namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 14a65876-5613-49f5-89ad-6988b8d0aee7 kind: Event lastTimestamp: "2026-06-02T16:01:15Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-02T16:01:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:01:15Z" name: message-dumper-raw-0d859-predictor-58bd984d44-v47p2.18b54efcd5541851 namespace: kserve-ci-e2e-test resourceVersion: "16975" uid: e200c756-1b8c-4b27-a39a-9d58d6f80ff5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-0d859-predictor-58bd984d44 namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: 4a5998b9-fe1d-4025-b4dc-1c248842f3c4 kind: Event lastTimestamp: "2026-06-02T15:59:30Z" message: 'Created pod: message-dumper-raw-0d859-predictor-58bd984d44-v47p2' metadata: creationTimestamp: "2026-06-02T15:59:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:59:30Z" name: message-dumper-raw-0d859-predictor-58bd984d44.18b54ee44486255e namespace: kserve-ci-e2e-test resourceVersion: "15996" uid: ae9389b5-9c37-4d54-b06a-948bdea5bdb3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-0d859-predictor namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: bfe403d2-a7ab-477e-b69c-35872e072577 kind: Event lastTimestamp: "2026-06-02T15:59:29Z" message: Scaled up replica set message-dumper-raw-0d859-predictor-58bd984d44 from 0 to 1 metadata: creationTimestamp: "2026-06-02T15:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:59:29Z" name: message-dumper-raw-0d859-predictor.18b54ee4412bc43f namespace: kserve-ci-e2e-test resourceVersion: "15981" uid: fab3eb60-13dc-4cca-8a2c-649d79571121 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T15:59:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-0d859-predictor namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 10949caf-1e83-4a32-88db-d74fdeeb9ab9 kind: Event lastTimestamp: "2026-06-02T15:59:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T15:59:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:59:59Z" name: message-dumper-raw-0d859-predictor.18b54ee7c062f79c namespace: kserve-ci-e2e-test resourceVersion: "16354" uid: 3ab51135-9b63-4b71-a3f9-a5a4500345dc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T15:59:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-0d859-predictor namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 10949caf-1e83-4a32-88db-d74fdeeb9ab9 kind: Event lastTimestamp: "2026-06-02T15:59:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T15:59:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:59:59Z" name: message-dumper-raw-0d859-predictor.18b54ee7c0dad946 namespace: kserve-ci-e2e-test resourceVersion: "16355" uid: 6041dd2f-4a46-4869-a4f9-c65afd1c1459 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-06-02T15:59:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-0d859 namespace: kserve-ci-e2e-test resourceVersion: "15957" uid: 85d1935d-b0b4-4785-bcd4-57b950482a14 kind: Event lastTimestamp: "2026-06-02T15:59:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-02T15:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:59:40Z" name: message-dumper-raw-0d859.18b54ee440faea10 namespace: kserve-ci-e2e-test resourceVersion: "16140" uid: 9cc4bcf6-d58d-4795-aad7-d97c133fb79a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T15:59:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-0d859 namespace: kserve-ci-e2e-test resourceVersion: "15957" uid: 85d1935d-b0b4-4785-bcd4-57b950482a14 kind: Event lastTimestamp: "2026-06-02T15:59:30Z" message: 'Failed to update status for InferenceService "message-dumper-raw-0d859": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-0d859": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-02T15:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:59:30Z" name: message-dumper-raw-0d859.18b54ee441db9e4e namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: e12e02f9-884a-4713-a3c9-61a0d742278e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-0d859 namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 85d1935d-b0b4-4785-bcd4-57b950482a14 kind: Event lastTimestamp: "2026-06-02T15:59:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-0d859": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-02T15:59:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:59:30Z" name: message-dumper-raw-0d859.18b54ee44836038f namespace: kserve-ci-e2e-test resourceVersion: "16019" uid: 41ca0dce-6a9e-48dd-8445-08024935701a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-0d859 namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 85d1935d-b0b4-4785-bcd4-57b950482a14 kind: Event lastTimestamp: "2026-06-02T15:59:40Z" message: InferenceService [message-dumper-raw-0d859] is Ready metadata: creationTimestamp: "2026-06-02T15:59:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:59:40Z" name: message-dumper-raw-0d859.18b54ee6bc8d996f namespace: kserve-ci-e2e-test resourceVersion: "16138" uid: 9517ac2d-59af-4d77-83ae-ed01370158aa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-02T15:57:39.090399Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-a7b79-79d5b46b87-dmfxm namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 60310fcf-4a9f-42bb-b713-536a434fee2e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-a7b79-79d5b46b87-dmfxm to ip-10-0-128-129.ec2.internal metadata: creationTimestamp: "2026-06-02T15:57:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-02T15:57:39Z" name: model-chainer-raw-a7b79-79d5b46b87-dmfxm.18b54eca718e55a1 namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: 1cd65ec9-e064-49b3-a643-35c09ead8837 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8ddcd7f6c-hs6d2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:39Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-a7b79-79d5b46b87-dmfxm namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: 60310fcf-4a9f-42bb-b713-536a434fee2e kind: Event lastTimestamp: "2026-06-02T15:57:39Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-02T15:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-02T15:57:39Z" name: model-chainer-raw-a7b79-79d5b46b87-dmfxm.18b54eca8b3e88d1 namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 49d9ef02-e7f5-4f54-901b-ff12b4e74451 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-a7b79} kind: Pod name: model-chainer-raw-a7b79-79d5b46b87-dmfxm namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 60310fcf-4a9f-42bb-b713-536a434fee2e kind: Event lastTimestamp: "2026-06-02T15:57:39Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:954d644696f600f3aaec773b2a281b1c1b7eb577f87d9494d88d7bc9da2d5a40" metadata: creationTimestamp: "2026-06-02T15:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:39Z" name: model-chainer-raw-a7b79-79d5b46b87-dmfxm.18b54eca8caa6e43 namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: 161ed71f-611a-42e7-a771-7c62e2ca20cb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-129.ec2.internal source: component: kubelet host: ip-10-0-128-129.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-a7b79} kind: Pod name: model-chainer-raw-a7b79-79d5b46b87-dmfxm namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 60310fcf-4a9f-42bb-b713-536a434fee2e kind: Event lastTimestamp: "2026-06-02T15:57:41Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:954d644696f600f3aaec773b2a281b1c1b7eb577f87d9494d88d7bc9da2d5a40" in 1.788s (1.788s including waiting). Image size: 218002526 bytes.' metadata: creationTimestamp: "2026-06-02T15:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:41Z" name: model-chainer-raw-a7b79-79d5b46b87-dmfxm.18b54ecaf741247a namespace: kserve-ci-e2e-test resourceVersion: "14884" uid: 4111ecf3-c188-4261-bbc3-6741b2959752 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-129.ec2.internal source: component: kubelet host: ip-10-0-128-129.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-a7b79} kind: Pod name: model-chainer-raw-a7b79-79d5b46b87-dmfxm namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 60310fcf-4a9f-42bb-b713-536a434fee2e kind: Event lastTimestamp: "2026-06-02T15:57:41Z" message: 'Created container: model-chainer-raw-a7b79' metadata: creationTimestamp: "2026-06-02T15:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:41Z" name: model-chainer-raw-a7b79-79d5b46b87-dmfxm.18b54ecafa4596be namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: 86d63d2e-2675-446d-a350-483d708d6148 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-129.ec2.internal source: component: kubelet host: ip-10-0-128-129.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-a7b79} kind: Pod name: model-chainer-raw-a7b79-79d5b46b87-dmfxm namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 60310fcf-4a9f-42bb-b713-536a434fee2e kind: Event lastTimestamp: "2026-06-02T15:57:41Z" message: Started container model-chainer-raw-a7b79 metadata: creationTimestamp: "2026-06-02T15:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:41Z" name: model-chainer-raw-a7b79-79d5b46b87-dmfxm.18b54ecafabdcf38 namespace: kserve-ci-e2e-test resourceVersion: "14887" uid: c399edeb-9eef-49d1-b28f-1c555ec82261 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-129.ec2.internal source: component: kubelet host: ip-10-0-128-129.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-a7b79} kind: Pod name: model-chainer-raw-a7b79-79d5b46b87-dmfxm namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 60310fcf-4a9f-42bb-b713-536a434fee2e kind: Event lastTimestamp: "2026-06-02T15:57:49Z" message: Stopping container model-chainer-raw-a7b79 metadata: creationTimestamp: "2026-06-02T15:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:57:49Z" name: model-chainer-raw-a7b79-79d5b46b87-dmfxm.18b54eccd27aa39f namespace: kserve-ci-e2e-test resourceVersion: "14971" uid: 3c6c9a66-f659-4a6e-8384-72eec7f1ab89 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-129.ec2.internal source: component: kubelet host: ip-10-0-128-129.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-02T15:57:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-a7b79} kind: Pod name: model-chainer-raw-a7b79-79d5b46b87-dmfxm namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 60310fcf-4a9f-42bb-b713-536a434fee2e kind: Event lastTimestamp: "2026-06-02T15:58:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-06-02T15:57:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:58:17Z" name: model-chainer-raw-a7b79-79d5b46b87-dmfxm.18b54ecd97a74454 namespace: kserve-ci-e2e-test resourceVersion: "15354" uid: 83b47415-ac39-4e33-bc4c-a71b9db260e5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-129.ec2.internal source: component: kubelet host: ip-10-0-128-129.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-a7b79-79d5b46b87 namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 4b75f78a-b95b-474e-8e25-1f1a3a53d9ed kind: Event lastTimestamp: "2026-06-02T15:57:39Z" message: 'Created pod: model-chainer-raw-a7b79-79d5b46b87-dmfxm' metadata: creationTimestamp: "2026-06-02T15:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:57:39Z" name: model-chainer-raw-a7b79-79d5b46b87.18b54eca711ad082 namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: 1ee2f383-0b04-4f2a-8514-501e217bd716 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-a7b79 namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: 5250c70b-c40a-4d7a-8aec-c78fc83ac450 kind: Event lastTimestamp: "2026-06-02T15:57:39Z" message: Scaled up replica set model-chainer-raw-a7b79-79d5b46b87 from 0 to 1 metadata: creationTimestamp: "2026-06-02T15:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:57:39Z" name: model-chainer-raw-a7b79.18b54eca7010d335 namespace: kserve-ci-e2e-test resourceVersion: "14842" uid: 2c71a210-6518-45af-9c8c-77e707935141 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-a7b79 namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 44072fae-00b7-4149-aeb1-c1c3cd9552ea kind: Event lastTimestamp: "2026-06-02T15:57:39Z" message: InferenceGraph [model-chainer-raw-a7b79] is Ready metadata: creationTimestamp: "2026-06-02T15:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:57:39Z" name: model-chainer-raw-a7b79.18b54eca784740c5 namespace: kserve-ci-e2e-test resourceVersion: "14860" uid: 02d7ce23-98a0-4031-b9ee-577a37350ff0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-a7b79 namespace: kserve-ci-e2e-test resourceVersion: "14832" uid: 44072fae-00b7-4149-aeb1-c1c3cd9552ea kind: Event lastTimestamp: "2026-06-02T15:57:39Z" message: 'Failed to update status for InferenceGraph "model-chainer-raw-a7b79": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-a7b79": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-02T15:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:57:39Z" name: model-chainer-raw-a7b79.18b54eca7b8e3d26 namespace: kserve-ci-e2e-test resourceVersion: "14862" uid: 160f61eb-608e-4db3-8644-fcaa4452b476 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:57:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-a7b79 namespace: kserve-ci-e2e-test resourceVersion: "14832" uid: 44072fae-00b7-4149-aeb1-c1c3cd9552ea kind: Event lastTimestamp: "2026-06-02T15:57:39Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-a7b79": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-02T15:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:57:39Z" name: model-chainer-raw-a7b79.18b54eca7b8e9abd namespace: kserve-ci-e2e-test resourceVersion: "14863" uid: 52e995a7-897d-45ca-9346-c43333d994a3 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-06-02T15:59:19.722072Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-ae9ae-6784d8f75f-hlxbk namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: a0bdeb1f-1f19-4491-9c6e-bdf057b0f90e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-ae9ae-6784d8f75f-hlxbk to ip-10-0-128-129.ec2.internal metadata: creationTimestamp: "2026-06-02T15:59:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-02T15:59:19Z" name: model-chainer-raw-hpa-ae9ae-6784d8f75f-hlxbk.18b54ee1dfabfa7e namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: 6cbbb948-c3d6-4df8-938d-7bf1d94b4c28 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8ddcd7f6c-hs6d2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:20Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-ae9ae-6784d8f75f-hlxbk namespace: kserve-ci-e2e-test resourceVersion: "15844" uid: a0bdeb1f-1f19-4491-9c6e-bdf057b0f90e kind: Event lastTimestamp: "2026-06-02T15:59:20Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-02T15:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-02T15:59:20Z" name: model-chainer-raw-hpa-ae9ae-6784d8f75f-hlxbk.18b54ee1f9329d8c namespace: kserve-ci-e2e-test resourceVersion: "15857" uid: d75276c1-989f-4dcf-bbe1-7259b97cee37 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-ae9ae} kind: Pod name: model-chainer-raw-hpa-ae9ae-6784d8f75f-hlxbk namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: a0bdeb1f-1f19-4491-9c6e-bdf057b0f90e kind: Event lastTimestamp: "2026-06-02T15:59:20Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:954d644696f600f3aaec773b2a281b1c1b7eb577f87d9494d88d7bc9da2d5a40" already present on machine metadata: creationTimestamp: "2026-06-02T15:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:20Z" name: model-chainer-raw-hpa-ae9ae-6784d8f75f-hlxbk.18b54ee1faac710f namespace: kserve-ci-e2e-test resourceVersion: "15859" uid: 9568472b-63ea-421b-8e4d-3f6fe5afd9bd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-129.ec2.internal source: component: kubelet host: ip-10-0-128-129.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-ae9ae} kind: Pod name: model-chainer-raw-hpa-ae9ae-6784d8f75f-hlxbk namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: a0bdeb1f-1f19-4491-9c6e-bdf057b0f90e kind: Event lastTimestamp: "2026-06-02T15:59:20Z" message: 'Created container: model-chainer-raw-hpa-ae9ae' metadata: creationTimestamp: "2026-06-02T15:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:20Z" name: model-chainer-raw-hpa-ae9ae-6784d8f75f-hlxbk.18b54ee1fdcc8179 namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: dfa3f9c7-54db-4dee-b200-6e3e46d6f5ea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-129.ec2.internal source: component: kubelet host: ip-10-0-128-129.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-ae9ae} kind: Pod name: model-chainer-raw-hpa-ae9ae-6784d8f75f-hlxbk namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: a0bdeb1f-1f19-4491-9c6e-bdf057b0f90e kind: Event lastTimestamp: "2026-06-02T15:59:20Z" message: Started container model-chainer-raw-hpa-ae9ae metadata: creationTimestamp: "2026-06-02T15:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:20Z" name: model-chainer-raw-hpa-ae9ae-6784d8f75f-hlxbk.18b54ee1fe465ec5 namespace: kserve-ci-e2e-test resourceVersion: "15861" uid: 043dae2b-1fbe-46c6-aba8-b3c40587bd30 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-129.ec2.internal source: component: kubelet host: ip-10-0-128-129.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-ae9ae} kind: Pod name: model-chainer-raw-hpa-ae9ae-6784d8f75f-hlxbk namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: a0bdeb1f-1f19-4491-9c6e-bdf057b0f90e kind: Event lastTimestamp: "2026-06-02T15:59:29Z" message: Stopping container model-chainer-raw-hpa-ae9ae metadata: creationTimestamp: "2026-06-02T15:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:29Z" name: model-chainer-raw-hpa-ae9ae-6784d8f75f-hlxbk.18b54ee4379953ad namespace: kserve-ci-e2e-test resourceVersion: "15953" uid: 81f0816d-7510-407c-9cdb-519cf4f76736 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-129.ec2.internal source: component: kubelet host: ip-10-0-128-129.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-06-02T15:59:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-ae9ae} kind: Pod name: model-chainer-raw-hpa-ae9ae-6784d8f75f-hlxbk namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: a0bdeb1f-1f19-4491-9c6e-bdf057b0f90e kind: Event lastTimestamp: "2026-06-02T15:59:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-06-02T15:59:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T15:59:56Z" name: model-chainer-raw-hpa-ae9ae-6784d8f75f-hlxbk.18b54ee4b8366df3 namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: 4b9325d6-84e9-432d-9746-d53ac68d0ee3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-129.ec2.internal source: component: kubelet host: ip-10-0-128-129.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-ae9ae-6784d8f75f namespace: kserve-ci-e2e-test resourceVersion: "15831" uid: 60eb99aa-e173-4aee-aaa4-c251484c33e8 kind: Event lastTimestamp: "2026-06-02T15:59:19Z" message: 'Created pod: model-chainer-raw-hpa-ae9ae-6784d8f75f-hlxbk' metadata: creationTimestamp: "2026-06-02T15:59:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:59:19Z" name: model-chainer-raw-hpa-ae9ae-6784d8f75f.18b54ee1df45174f namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 69218f59-bc0e-4259-a27c-9add7cfb52cf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-ae9ae namespace: kserve-ci-e2e-test resourceVersion: "15829" uid: 0adc7dac-d9d9-4653-92cb-7a1dc9c97f29 kind: Event lastTimestamp: "2026-06-02T15:59:19Z" message: Scaled up replica set model-chainer-raw-hpa-ae9ae-6784d8f75f from 0 to 1 metadata: creationTimestamp: "2026-06-02T15:59:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T15:59:19Z" name: model-chainer-raw-hpa-ae9ae.18b54ee1de2e9736 namespace: kserve-ci-e2e-test resourceVersion: "15832" uid: 9bda1c4a-c908-4f45-ae65-3e8a9b0786b4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T15:59:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-ae9ae namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: 2c30dd0b-6229-4d4e-a873-248e5c87aef6 kind: Event lastTimestamp: "2026-06-02T15:59:19Z" message: InferenceGraph [model-chainer-raw-hpa-ae9ae] is Ready metadata: creationTimestamp: "2026-06-02T15:59:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T15:59:19Z" name: model-chainer-raw-hpa-ae9ae.18b54ee1e2e76b35 namespace: kserve-ci-e2e-test resourceVersion: "15850" uid: 0985a4b5-2fcb-4553-ac59-e401974ae4cc reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-02T16:05:24.196770Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p namespace: kserve-ci-e2e-test resourceVersion: "19095" uid: 65d5bbcf-f448-4c14-94d5-b2f83613cacb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p to ip-10-0-140-246.ec2.internal metadata: creationTimestamp: "2026-06-02T16:05:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-02T16:05:24Z" name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p.18b54f36bc0e4dda namespace: kserve-ci-e2e-test resourceVersion: "19100" uid: 1eade3d1-1f46-4977-8326-cdcff003d173 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8ddcd7f6c-hs6d2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:24Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p namespace: kserve-ci-e2e-test resourceVersion: "19106" uid: 65d5bbcf-f448-4c14-94d5-b2f83613cacb kind: Event lastTimestamp: "2026-06-02T16:05:24Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-02T16:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-02T16:05:24Z" name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p.18b54f36d64fd3b5 namespace: kserve-ci-e2e-test resourceVersion: "19111" uid: b07f9784-ac5e-42bc-8b00-242b7d49c177 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p namespace: kserve-ci-e2e-test resourceVersion: "19098" uid: 65d5bbcf-f448-4c14-94d5-b2f83613cacb kind: Event lastTimestamp: "2026-06-02T16:05:24Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ca9e7e1d07975dffacc4b293d655f2a02df4615629916a1cdc022036c5a27a57" already present on machine metadata: creationTimestamp: "2026-06-02T16:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:24Z" name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p.18b54f36d7d7f979 namespace: kserve-ci-e2e-test resourceVersion: "19113" uid: 1a782df2-74c0-43f5-ad64-8671de5d09c7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p namespace: kserve-ci-e2e-test resourceVersion: "19098" uid: 65d5bbcf-f448-4c14-94d5-b2f83613cacb kind: Event lastTimestamp: "2026-06-02T16:05:24Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-02T16:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:24Z" name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p.18b54f36dbb81d48 namespace: kserve-ci-e2e-test resourceVersion: "19114" uid: 85cc6389-40ce-46b8-b8d7-6b6c1135398e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p namespace: kserve-ci-e2e-test resourceVersion: "19098" uid: 65d5bbcf-f448-4c14-94d5-b2f83613cacb kind: Event lastTimestamp: "2026-06-02T16:05:24Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-02T16:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:24Z" name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p.18b54f36dc2ae26d namespace: kserve-ci-e2e-test resourceVersion: "19115" uid: bf2096c5-e8e6-4278-9f1a-2c52afd18710 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p namespace: kserve-ci-e2e-test resourceVersion: "19098" uid: 65d5bbcf-f448-4c14-94d5-b2f83613cacb kind: Event lastTimestamp: "2026-06-02T16:05:28Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1560" already present on machine metadata: creationTimestamp: "2026-06-02T16:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:28Z" name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p.18b54f37c3d886dc namespace: kserve-ci-e2e-test resourceVersion: "19153" uid: dcbd282f-96bb-4dd9-a584-16a13a7bd7a7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p namespace: kserve-ci-e2e-test resourceVersion: "19098" uid: 65d5bbcf-f448-4c14-94d5-b2f83613cacb kind: Event lastTimestamp: "2026-06-02T16:05:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-02T16:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:28Z" name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p.18b54f37c6fceba3 namespace: kserve-ci-e2e-test resourceVersion: "19158" uid: b536ad33-35a1-417f-ac59-bd03ce0a9e4e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p namespace: kserve-ci-e2e-test resourceVersion: "19098" uid: 65d5bbcf-f448-4c14-94d5-b2f83613cacb kind: Event lastTimestamp: "2026-06-02T16:05:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-02T16:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:28Z" name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p.18b54f37c7aa8a58 namespace: kserve-ci-e2e-test resourceVersion: "19159" uid: 202497ee-e4ec-4fc5-b183-7e2c51098144 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p namespace: kserve-ci-e2e-test resourceVersion: "19098" uid: 65d5bbcf-f448-4c14-94d5-b2f83613cacb kind: Event lastTimestamp: "2026-06-02T16:05:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-02T16:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:28Z" name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p.18b54f37c7bb64b8 namespace: kserve-ci-e2e-test resourceVersion: "19160" uid: 62fbeed3-ea15-41cb-a104-3856f6f31a50 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p namespace: kserve-ci-e2e-test resourceVersion: "19098" uid: 65d5bbcf-f448-4c14-94d5-b2f83613cacb kind: Event lastTimestamp: "2026-06-02T16:05:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-02T16:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:28Z" name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p.18b54f37cab35b97 namespace: kserve-ci-e2e-test resourceVersion: "19163" uid: e791f073-409f-4ac5-ab74-05ad64b2b0f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p namespace: kserve-ci-e2e-test resourceVersion: "19098" uid: 65d5bbcf-f448-4c14-94d5-b2f83613cacb kind: Event lastTimestamp: "2026-06-02T16:05:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T16:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:05:28Z" name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p.18b54f37cb62b87b namespace: kserve-ci-e2e-test resourceVersion: "19165" uid: e20fb232-e0cd-49e7-9a1a-a2fdd56ae113 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-06-02T16:05:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p namespace: kserve-ci-e2e-test resourceVersion: "19098" uid: 65d5bbcf-f448-4c14-94d5-b2f83613cacb kind: Event lastTimestamp: "2026-06-02T16:06:46Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-02T16:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:06:46Z" name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p.18b54f383be38e29 namespace: kserve-ci-e2e-test resourceVersion: "19792" uid: 8dd3d05a-5259-4edc-a156-19280a04239a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:06:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p namespace: kserve-ci-e2e-test resourceVersion: "19098" uid: 65d5bbcf-f448-4c14-94d5-b2f83613cacb kind: Event lastTimestamp: "2026-06-02T16:06:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-02T16:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:06:44Z" name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p.18b54f496167aee4 namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: aec36f5c-3ce0-4a9a-a099-b762e9e8bb2c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:06:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p namespace: kserve-ci-e2e-test resourceVersion: "19098" uid: 65d5bbcf-f448-4c14-94d5-b2f83613cacb kind: Event lastTimestamp: "2026-06-02T16:06:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T16:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:06:44Z" name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p.18b54f49616852a0 namespace: kserve-ci-e2e-test resourceVersion: "19733" uid: ee539a0e-cd29-4aad-8380-87c5115f9aa4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:06:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p namespace: kserve-ci-e2e-test resourceVersion: "19098" uid: 65d5bbcf-f448-4c14-94d5-b2f83613cacb kind: Event lastTimestamp: "2026-06-02T16:06:46Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-02T16:06:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:06:46Z" name: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p.18b54f49ee008155 namespace: kserve-ci-e2e-test resourceVersion: "19791" uid: ba25f17b-1b2e-4864-9a17-9782be1eccb3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-a5de4-predictor-cd89bbb98 namespace: kserve-ci-e2e-test resourceVersion: "19084" uid: 5f9e9677-e7bb-4560-a339-3fd85d258ccb kind: Event lastTimestamp: "2026-06-02T16:05:24Z" message: 'Created pod: raw-sklearn-a5de4-predictor-cd89bbb98-8cl4p' metadata: creationTimestamp: "2026-06-02T16:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:05:24Z" name: raw-sklearn-a5de4-predictor-cd89bbb98.18b54f36bb925142 namespace: kserve-ci-e2e-test resourceVersion: "19097" uid: 7bdf4a7d-b03d-492c-8cb1-ab29f37f55c2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-a5de4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19082" uid: bb36f404-dbcf-4ba3-bcba-2627ff5cd70b kind: Event lastTimestamp: "2026-06-02T16:05:24Z" message: Scaled up replica set raw-sklearn-a5de4-predictor-cd89bbb98 from 0 to 1 metadata: creationTimestamp: "2026-06-02T16:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:05:24Z" name: raw-sklearn-a5de4-predictor.18b54f36b7d096f6 namespace: kserve-ci-e2e-test resourceVersion: "19086" uid: 2decd8d8-e1aa-4528-9ed6-380b75c58da5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-02T16:05:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-a5de4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19083" uid: 99bb4051-7099-4322-8901-6a87fc6c3c0b kind: Event lastTimestamp: "2026-06-02T16:06:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T16:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:06:09Z" name: raw-sklearn-a5de4-predictor.18b54f3a36bf6ce4 namespace: kserve-ci-e2e-test resourceVersion: "19452" uid: a22a8d77-ffe1-43db-b9d8-2b6dffc44392 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-02T16:05:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-a5de4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19083" uid: 99bb4051-7099-4322-8901-6a87fc6c3c0b kind: Event lastTimestamp: "2026-06-02T16:06:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T16:05:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:06:09Z" name: raw-sklearn-a5de4-predictor.18b54f3a3729948f namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: 5180dbe1-7996-410e-92d9-8278f88384eb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T16:06:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-a5de4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19243" uid: 99bb4051-7099-4322-8901-6a87fc6c3c0b kind: Event lastTimestamp: "2026-06-02T16:06:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-02T16:06:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:06:39Z" name: raw-sklearn-a5de4-predictor.18b54f44b461a12e namespace: kserve-ci-e2e-test resourceVersion: "19671" uid: 7111674b-e3d3-4cc0-8f40-80bda4e251dc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T16:06:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-a5de4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19243" uid: 99bb4051-7099-4322-8901-6a87fc6c3c0b kind: Event lastTimestamp: "2026-06-02T16:06:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-02T16:06:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:06:39Z" name: raw-sklearn-a5de4-predictor.18b54f44b4ccdfa0 namespace: kserve-ci-e2e-test resourceVersion: "19672" uid: 8b6aa531-e30b-4b82-a58f-a70c84146794 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-06-02T16:05:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-a5de4 namespace: kserve-ci-e2e-test resourceVersion: "19069" uid: 28d843fe-ed0f-4f97-b3cf-e46cf5b9b96d kind: Event lastTimestamp: "2026-06-02T16:06:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-02T16:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:06:36Z" name: raw-sklearn-a5de4.18b54f36b7accade namespace: kserve-ci-e2e-test resourceVersion: "19655" uid: 7b600065-fe36-41fd-bba6-255ea7764847 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:05:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-a5de4 namespace: kserve-ci-e2e-test resourceVersion: "19069" uid: 28d843fe-ed0f-4f97-b3cf-e46cf5b9b96d kind: Event lastTimestamp: "2026-06-02T16:05:24Z" message: 'Failed to update status for InferenceService "raw-sklearn-a5de4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-a5de4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-02T16:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:05:24Z" name: raw-sklearn-a5de4.18b54f36b86be8c5 namespace: kserve-ci-e2e-test resourceVersion: "19088" uid: 1da056b6-892d-4d5a-919e-a9a70a153a0c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:06:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-a5de4 namespace: kserve-ci-e2e-test resourceVersion: "19650" uid: 28d843fe-ed0f-4f97-b3cf-e46cf5b9b96d kind: Event lastTimestamp: "2026-06-02T16:06:36Z" message: InferenceService [raw-sklearn-a5de4] is Ready metadata: creationTimestamp: "2026-06-02T16:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:06:36Z" name: raw-sklearn-a5de4.18b54f47a0873210 namespace: kserve-ci-e2e-test resourceVersion: "19651" uid: 24a20b43-97ca-46b0-94d2-8a7320426aca reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-06-02T16:06:44.376417Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: 216f876a-b63e-4ce3-a566-20ac5cd45371 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc to ip-10-0-140-246.ec2.internal metadata: creationTimestamp: "2026-06-02T16:06:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-06-02T16:06:44Z" name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc.18b54f49672295d9 namespace: kserve-ci-e2e-test resourceVersion: "19754" uid: f91ee169-68ee-4ac8-b5b2-6a2f21ec4c54 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-8ddcd7f6c-hs6d2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:06:44Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc namespace: kserve-ci-e2e-test resourceVersion: "19759" uid: 216f876a-b63e-4ce3-a566-20ac5cd45371 kind: Event lastTimestamp: "2026-06-02T16:06:44Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-06-02T16:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-06-02T16:06:44Z" name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc.18b54f49805d61d8 namespace: kserve-ci-e2e-test resourceVersion: "19766" uid: b490b6c4-8b12-4af9-a9ca-3a4107dab757 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:06:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 216f876a-b63e-4ce3-a566-20ac5cd45371 kind: Event lastTimestamp: "2026-06-02T16:06:44Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ca9e7e1d07975dffacc4b293d655f2a02df4615629916a1cdc022036c5a27a57" already present on machine metadata: creationTimestamp: "2026-06-02T16:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:06:44Z" name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc.18b54f4981e2327f namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 778d91e4-37ae-4ca3-b81b-66698be5aa09 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:06:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 216f876a-b63e-4ce3-a566-20ac5cd45371 kind: Event lastTimestamp: "2026-06-02T16:06:44Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-06-02T16:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:06:44Z" name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc.18b54f4984b0836b namespace: kserve-ci-e2e-test resourceVersion: "19770" uid: 46c8f474-be89-433c-913d-f3093b6314d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:06:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 216f876a-b63e-4ce3-a566-20ac5cd45371 kind: Event lastTimestamp: "2026-06-02T16:06:44Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-06-02T16:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:06:44Z" name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc.18b54f49850de8eb namespace: kserve-ci-e2e-test resourceVersion: "19771" uid: b1b421d9-57b0-409e-854f-9fbc06a12c20 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:06:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 216f876a-b63e-4ce3-a566-20ac5cd45371 kind: Event lastTimestamp: "2026-06-02T16:06:48Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1560" already present on machine metadata: creationTimestamp: "2026-06-02T16:06:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:06:48Z" name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc.18b54f4a732f7f1b namespace: kserve-ci-e2e-test resourceVersion: "19811" uid: a089e895-83cb-44ea-a9d3-3b2e22da0262 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:06:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 216f876a-b63e-4ce3-a566-20ac5cd45371 kind: Event lastTimestamp: "2026-06-02T16:06:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-06-02T16:06:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:06:48Z" name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc.18b54f4a76cbdddd namespace: kserve-ci-e2e-test resourceVersion: "19815" uid: 6a1d569c-d389-42e7-82d2-008c40ba9b19 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:06:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 216f876a-b63e-4ce3-a566-20ac5cd45371 kind: Event lastTimestamp: "2026-06-02T16:06:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-06-02T16:06:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:06:48Z" name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc.18b54f4a77595cdb namespace: kserve-ci-e2e-test resourceVersion: "19817" uid: a519da94-b1d8-468c-82de-1e6d29f81efe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:06:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 216f876a-b63e-4ce3-a566-20ac5cd45371 kind: Event lastTimestamp: "2026-06-02T16:06:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-06-02T16:06:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:06:48Z" name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc.18b54f4a7765e9b3 namespace: kserve-ci-e2e-test resourceVersion: "19818" uid: 05aa9dbe-8873-4bdf-aeec-162b99489735 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:06:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 216f876a-b63e-4ce3-a566-20ac5cd45371 kind: Event lastTimestamp: "2026-06-02T16:06:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-06-02T16:06:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:06:48Z" name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc.18b54f4a7a103459 namespace: kserve-ci-e2e-test resourceVersion: "19821" uid: 879722eb-66c2-4666-9434-69f2302122c8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:06:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 216f876a-b63e-4ce3-a566-20ac5cd45371 kind: Event lastTimestamp: "2026-06-02T16:06:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T16:06:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:06:48Z" name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc.18b54f4a7a7e2b09 namespace: kserve-ci-e2e-test resourceVersion: "19822" uid: 385d21aa-313c-451c-8325-29b45b5a86ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-06-02T16:06:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 216f876a-b63e-4ce3-a566-20ac5cd45371 kind: Event lastTimestamp: "2026-06-02T16:08:06Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-06-02T16:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:08:06Z" name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc.18b54f4aeaed2641 namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 3ffa5a90-9fd2-49ae-897d-26e195bbf962 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:08:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 216f876a-b63e-4ce3-a566-20ac5cd45371 kind: Event lastTimestamp: "2026-06-02T16:08:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-06-02T16:08:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:08:04Z" name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc.18b54f5c159fea25 namespace: kserve-ci-e2e-test resourceVersion: "20390" uid: bd34904f-bb4a-4cbb-a34a-4cd55dfe2edb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:08:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 216f876a-b63e-4ce3-a566-20ac5cd45371 kind: Event lastTimestamp: "2026-06-02T16:08:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-06-02T16:08:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:08:04Z" name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc.18b54f5c15a0802e namespace: kserve-ci-e2e-test resourceVersion: "20391" uid: ab256a29-fa94-4374-8ca4-836436584952 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:08:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 216f876a-b63e-4ce3-a566-20ac5cd45371 kind: Event lastTimestamp: "2026-06-02T16:08:06Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-06-02T16:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-06-02T16:08:06Z" name: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc.18b54f5c9d013358 namespace: kserve-ci-e2e-test resourceVersion: "20440" uid: 3acc415c-d0ce-4c0f-bfb8-408394972eb7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-246.ec2.internal source: component: kubelet host: ip-10-0-140-246.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:06:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-c84fb-predictor-68c976fd68 namespace: kserve-ci-e2e-test resourceVersion: "19738" uid: 8bcbdf0b-a9ee-4676-98c8-2d1f239846e8 kind: Event lastTimestamp: "2026-06-02T16:06:44Z" message: 'Created pod: raw-sklearn-runtime-c84fb-predictor-68c976fd68-xd6jc' metadata: creationTimestamp: "2026-06-02T16:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:06:44Z" name: raw-sklearn-runtime-c84fb-predictor-68c976fd68.18b54f49668d9928 namespace: kserve-ci-e2e-test resourceVersion: "19751" uid: 1ad49c6f-662a-47c3-8c35-3a4904002258 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:06:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-c84fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "19736" uid: ab6bdca8-0312-40e4-a3ea-e67d1b257aaf kind: Event lastTimestamp: "2026-06-02T16:06:44Z" message: Scaled up replica set raw-sklearn-runtime-c84fb-predictor-68c976fd68 from 0 to 1 metadata: creationTimestamp: "2026-06-02T16:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:06:44Z" name: raw-sklearn-runtime-c84fb-predictor.18b54f4963369862 namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: 785ddbf8-b734-4e47-83ee-39ed3b4b8685 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T16:06:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-c84fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "19737" uid: 66121a1f-0cdc-41a3-a7e0-fa65b19146ef kind: Event lastTimestamp: "2026-06-02T16:07:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T16:06:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:07:14Z" name: raw-sklearn-runtime-c84fb-predictor.18b54f4ce1d921bb namespace: kserve-ci-e2e-test resourceVersion: "20007" uid: e0dbc033-9260-476a-aa38-12ae77174706 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-06-02T16:06:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-c84fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "19737" uid: 66121a1f-0cdc-41a3-a7e0-fa65b19146ef kind: Event lastTimestamp: "2026-06-02T16:07:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-06-02T16:06:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:07:14Z" name: raw-sklearn-runtime-c84fb-predictor.18b54f4ce24c2d36 namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: c83cbea1-1b3e-4b44-94ae-e0f220459c6f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-02T16:07:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-c84fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "19900" uid: 66121a1f-0cdc-41a3-a7e0-fa65b19146ef kind: Event lastTimestamp: "2026-06-02T16:07:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-02T16:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:07:59Z" name: raw-sklearn-runtime-c84fb-predictor.18b54f53e07f8789 namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: 23b4c997-5103-45bc-8708-d4a4ebd462c9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-06-02T16:07:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-c84fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "19900" uid: 66121a1f-0cdc-41a3-a7e0-fa65b19146ef kind: Event lastTimestamp: "2026-06-02T16:07:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-06-02T16:07:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-06-02T16:07:59Z" name: raw-sklearn-runtime-c84fb-predictor.18b54f53e0f859bd namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: 4f259aea-cbf8-44fe-aa34-1f91ed2baf4b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-06-02T16:06:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-c84fb namespace: kserve-ci-e2e-test resourceVersion: "19708" uid: 86fb9c84-5f61-462e-bb11-e332dc210cf5 kind: Event lastTimestamp: "2026-06-02T16:07:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-06-02T16:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:07:57Z" name: raw-sklearn-runtime-c84fb.18b54f496311c2d9 namespace: kserve-ci-e2e-test resourceVersion: "20318" uid: 75a7113d-2fde-4749-94ce-d6de5da9f5f8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:06:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-c84fb namespace: kserve-ci-e2e-test resourceVersion: "19708" uid: 86fb9c84-5f61-462e-bb11-e332dc210cf5 kind: Event lastTimestamp: "2026-06-02T16:06:44Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-c84fb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-c84fb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-06-02T16:06:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:06:44Z" name: raw-sklearn-runtime-c84fb.18b54f496383b450 namespace: kserve-ci-e2e-test resourceVersion: "19742" uid: ae7122e5-d53a-46f3-9a44-b7e50adbcd41 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-06-02T16:07:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-c84fb namespace: kserve-ci-e2e-test resourceVersion: "20316" uid: 86fb9c84-5f61-462e-bb11-e332dc210cf5 kind: Event lastTimestamp: "2026-06-02T16:07:56Z" message: InferenceService [raw-sklearn-runtime-c84fb] is Ready metadata: creationTimestamp: "2026-06-02T16:07:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-06-02T16:07:56Z" name: raw-sklearn-runtime-c84fb.18b54f5a4ef05da5 namespace: kserve-ci-e2e-test resourceVersion: "20317" uid: 6ba4842d-86b8-4741-aab9-357e19b8eea6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "20596"