--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:08:49.663451Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-bb9d97-predictor-64cb7bd4b5-t2p6k namespace: kserve-ci-e2e-test resourceVersion: "19200" uid: 75adfcb7-d701-40c1-8454-d83299a69a8e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-bb9d97-predictor-64cb7bd4b5-t2p6k to ip-10-0-140-251.ec2.internal metadata: creationTimestamp: "2026-05-20T14:08:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:08:49Z" name: isvc-init-fail-bb9d97-predictor-64cb7bd4b5-t2p6k.18b14b4ed05ecbc2 namespace: kserve-ci-e2e-test resourceVersion: "19204" uid: 1f5d706e-47d1-4df6-87c9-85cd93269db8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-c8868bc95-7kctc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:08:49Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-bb9d97-predictor-64cb7bd4b5-t2p6k namespace: kserve-ci-e2e-test resourceVersion: "19202" uid: 75adfcb7-d701-40c1-8454-d83299a69a8e kind: Event lastTimestamp: "2026-05-20T14:08:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-init-fail-bb9d97-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:08:49Z" name: isvc-init-fail-bb9d97-predictor-64cb7bd4b5-t2p6k.18b14b4ee20efbe7 namespace: kserve-ci-e2e-test resourceVersion: "19221" uid: c44c2ea4-1c9d-4808-9847-a2b33cd43342 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:08:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-bb9d97-predictor-64cb7bd4b5-t2p6k namespace: kserve-ci-e2e-test resourceVersion: "19210" uid: 75adfcb7-d701-40c1-8454-d83299a69a8e kind: Event lastTimestamp: "2026-05-20T14:08:50Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:08:50Z" name: isvc-init-fail-bb9d97-predictor-64cb7bd4b5-t2p6k.18b14b4f0e83a2aa namespace: kserve-ci-e2e-test resourceVersion: "19231" uid: 29da976d-8081-481e-b8e3-3aadfd157ee9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:08:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-bb9d97-predictor-64cb7bd4b5-t2p6k namespace: kserve-ci-e2e-test resourceVersion: "19202" uid: 75adfcb7-d701-40c1-8454-d83299a69a8e kind: Event lastTimestamp: "2026-05-20T14:08:55Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0459e31fff147b1f9487117ac00a86221572ce2251d3068d36cf76ed2126ea4f" already present on machine metadata: creationTimestamp: "2026-05-20T14:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:08:55Z" name: isvc-init-fail-bb9d97-predictor-64cb7bd4b5-t2p6k.18b14b4f0fec85a1 namespace: kserve-ci-e2e-test resourceVersion: "19280" uid: e0f0f808-6e95-47b3-a618-8f342f1e3ae9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:08:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-bb9d97-predictor-64cb7bd4b5-t2p6k namespace: kserve-ci-e2e-test resourceVersion: "19202" uid: 75adfcb7-d701-40c1-8454-d83299a69a8e kind: Event lastTimestamp: "2026-05-20T14:08:55Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T14:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:08:55Z" name: isvc-init-fail-bb9d97-predictor-64cb7bd4b5-t2p6k.18b14b4f131c6b9f namespace: kserve-ci-e2e-test resourceVersion: "19282" uid: 18513093-f964-45dc-a8d0-98b2c9e21456 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:08:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-bb9d97-predictor-64cb7bd4b5-t2p6k namespace: kserve-ci-e2e-test resourceVersion: "19202" uid: 75adfcb7-d701-40c1-8454-d83299a69a8e kind: Event lastTimestamp: "2026-05-20T14:08:56Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T14:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:08:56Z" name: isvc-init-fail-bb9d97-predictor-64cb7bd4b5-t2p6k.18b14b4f139b147b namespace: kserve-ci-e2e-test resourceVersion: "19284" uid: 511e3549-2982-4a26-9f2a-f6eefced89e7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:08:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-bb9d97-predictor-64cb7bd4b5-t2p6k namespace: kserve-ci-e2e-test resourceVersion: "19202" uid: 75adfcb7-d701-40c1-8454-d83299a69a8e kind: Event lastTimestamp: "2026-05-20T14:08:59Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-05-20T14:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:08:59Z" name: isvc-init-fail-bb9d97-predictor-64cb7bd4b5-t2p6k.18b14b5123d2221d namespace: kserve-ci-e2e-test resourceVersion: "19335" uid: 2c650843-0238-4da4-86b5-55fe17300ff8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:08:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-bb9d97-predictor-64cb7bd4b5 namespace: kserve-ci-e2e-test resourceVersion: "19192" uid: 993ea39c-c257-43c1-b616-80b60dd3d0c8 kind: Event lastTimestamp: "2026-05-20T14:08:49Z" message: 'Created pod: isvc-init-fail-bb9d97-predictor-64cb7bd4b5-t2p6k' metadata: creationTimestamp: "2026-05-20T14:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:08:49Z" name: isvc-init-fail-bb9d97-predictor-64cb7bd4b5.18b14b4ecfcc57dd namespace: kserve-ci-e2e-test resourceVersion: "19201" uid: cca808dd-6a31-4a71-b651-2f39aa61ae33 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:08:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-bb9d97-predictor namespace: kserve-ci-e2e-test resourceVersion: "19190" uid: b76cd8a0-d8d8-46db-aec0-097f4ca0d2bd kind: Event lastTimestamp: "2026-05-20T14:08:49Z" message: Scaled up replica set isvc-init-fail-bb9d97-predictor-64cb7bd4b5 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:08:49Z" name: isvc-init-fail-bb9d97-predictor.18b14b4eccf8ae94 namespace: kserve-ci-e2e-test resourceVersion: "19194" uid: 213279b7-3591-456b-aa03-f60a5a78eff4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-20T14:08:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-bb9d97 namespace: kserve-ci-e2e-test resourceVersion: "19163" uid: c68517b9-83f8-4a65-9564-ab4f6b4af641 kind: Event lastTimestamp: "2026-05-20T14:08:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:08:56Z" name: isvc-init-fail-bb9d97.18b14b4eccd59d7c namespace: kserve-ci-e2e-test resourceVersion: "19286" uid: 7af5ab2e-7ec3-480b-aa79-f367a224c369 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:08:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-bb9d97 namespace: kserve-ci-e2e-test resourceVersion: "19163" uid: c68517b9-83f8-4a65-9564-ab4f6b4af641 kind: Event lastTimestamp: "2026-05-20T14:08:49Z" message: 'Failed to update status for InferenceService "isvc-init-fail-bb9d97": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-bb9d97": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:08:49Z" name: isvc-init-fail-bb9d97.18b14b4ecd505f56 namespace: kserve-ci-e2e-test resourceVersion: "19213" uid: de0a4f3f-8d24-4138-91b6-ba03e08ed8b3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:08:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-bb9d97 namespace: kserve-ci-e2e-test resourceVersion: "19163" uid: c68517b9-83f8-4a65-9564-ab4f6b4af641 kind: Event lastTimestamp: "2026-05-20T14:08:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-bb9d97": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:08:49Z" name: isvc-init-fail-bb9d97.18b14b4ecd50c0c6 namespace: kserve-ci-e2e-test resourceVersion: "19214" uid: 67f4a6da-d8b7-40ca-819a-cf7848be09eb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:03:19.856254Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp to ip-10-0-140-251.ec2.internal metadata: creationTimestamp: "2026-05-20T14:03:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:03:19Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b0206545e60 namespace: kserve-ci-e2e-test resourceVersion: "16472" uid: dc89c68a-998f-4478-b1f2-7a2d853a7c33 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-c8868bc95-7kctc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: "2026-05-20T14:03:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-raw-cbb25-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:20Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b02140f3b49 namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 473f24b7-e529-43c2-beaa-931434a3879d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16476" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: "2026-05-20T14:03:20Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:03:20Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b02445a7dfe namespace: kserve-ci-e2e-test resourceVersion: "16490" uid: fad60682-3355-44e1-bde0-9871ceea1484 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: "2026-05-20T14:03:20Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0459e31fff147b1f9487117ac00a86221572ce2251d3068d36cf76ed2126ea4f" already present on machine metadata: creationTimestamp: "2026-05-20T14:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:20Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b0245f39c97 namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: f2dcce41-9f01-4503-90fb-93415b8dfa1c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: "2026-05-20T14:03:20Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T14:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:20Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b024903647c namespace: kserve-ci-e2e-test resourceVersion: "16495" uid: c11113a5-b3fe-4b34-a25a-14336ca8a659 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: "2026-05-20T14:03:20Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T14:03:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:20Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b024986cb27 namespace: kserve-ci-e2e-test resourceVersion: "16496" uid: f1d91706-673d-4346-9cbe-8b60c2d56a32 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: "2026-05-20T14:03:24Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:03:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:24Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b032f04f830 namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 89c1eac2-891c-42e6-a3e6-c0fca2abb0a4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: "2026-05-20T14:03:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:03:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:24Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b033266ccce namespace: kserve-ci-e2e-test resourceVersion: "16537" uid: a7d3054f-af9a-49fb-b444-762ac7caca50 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: "2026-05-20T14:03:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:03:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:24Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b0332ed5eb5 namespace: kserve-ci-e2e-test resourceVersion: "16538" uid: 63e0b3e1-98fb-408a-9b75-3e1cd9c2964c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: "2026-05-20T14:03:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:03:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:24Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b0332fd2cae namespace: kserve-ci-e2e-test resourceVersion: "16539" uid: 2883bac1-9a7b-43d8-aab5-a5b9fc1ed400 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: "2026-05-20T14:03:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:03:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:24Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b0336808f83 namespace: kserve-ci-e2e-test resourceVersion: "16541" uid: 29b418b2-8485-4750-9053-6d91ee54b6b1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: "2026-05-20T14:03:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:03:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:24Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b033708b727 namespace: kserve-ci-e2e-test resourceVersion: "16542" uid: d3655f07-432b-4c31-ac53-68fcc79fdb45 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: "2026-05-20T14:03:24Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:093b5585535b7b4890c43b46d80238c68db97d853fe4f3a7840d67abf2e0a4b5" already present on machine metadata: creationTimestamp: "2026-05-20T14:03:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:24Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b033715ca43 namespace: kserve-ci-e2e-test resourceVersion: "16543" uid: a0f700ac-da97-45c5-a4b6-fa2aad819ed3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: "2026-05-20T14:03:25Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-05-20T14:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:25Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b033a395dda namespace: kserve-ci-e2e-test resourceVersion: "16545" uid: 6537f7c8-b292-4369-9d39-ca4e61898bad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: "2026-05-20T14:03:25Z" message: Started container agent metadata: creationTimestamp: "2026-05-20T14:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:25Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b033b0b1320 namespace: kserve-ci-e2e-test resourceVersion: "16546" uid: 6b1060e4-51e4-4bf2-9000-6b7e666c09a5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-05-20T14:03:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: "2026-05-20T14:05:02Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:03:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:05:02Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b036af77e47 namespace: kserve-ci-e2e-test resourceVersion: "17371" uid: 132d842d-5bdc-4d13-a866-aa019224088e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T14:03:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: "2026-05-20T14:04:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T14:03:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:52Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b03a6b5f08c namespace: kserve-ci-e2e-test resourceVersion: "17295" uid: 0aef97ea-1ee1-4ec1-b011-d1becd6537d8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: "2026-05-20T14:04:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:45Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b15dc5a4591 namespace: kserve-ci-e2e-test resourceVersion: "17179" uid: 6e8b8f49-9a46-4793-89b3-9cee3213a6f9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: "2026-05-20T14:04:45Z" message: Stopping container agent metadata: creationTimestamp: "2026-05-20T14:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:45Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b15dc5a6e61 namespace: kserve-ci-e2e-test resourceVersion: "17180" uid: 062393ee-762c-4cd6-87ff-63e003020d1c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: "2026-05-20T14:04:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:45Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b15dc5adb35 namespace: kserve-ci-e2e-test resourceVersion: "17181" uid: b4a287a9-4297-471b-b88b-061f42016993 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:04:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: e2977e91-a2b4-4808-8558-4184e458a9b2 kind: Event lastTimestamp: "2026-05-20T14:05:02Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:04:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:05:02Z" name: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp.18b14b1682ea3994 namespace: kserve-ci-e2e-test resourceVersion: "17370" uid: 6efcd308-1577-4a49-bbc3-b0e1b628c8e4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-cbb25-predictor-5959584f64 namespace: kserve-ci-e2e-test resourceVersion: "16455" uid: 15664de4-ff2d-41fe-91dd-d8d998182dc5 kind: Event lastTimestamp: "2026-05-20T14:03:19Z" message: 'Created pod: isvc-logger-raw-cbb25-predictor-5959584f64-9zrpp' metadata: creationTimestamp: "2026-05-20T14:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:19Z" name: isvc-logger-raw-cbb25-predictor-5959584f64.18b14b0205ba3cba namespace: kserve-ci-e2e-test resourceVersion: "16468" uid: c4d0aba7-5bd4-4ebd-a065-08842a6a8d4a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-cbb25-predictor namespace: kserve-ci-e2e-test resourceVersion: "16453" uid: 092e8462-cf7f-4cca-8bc0-1aec3a8b779e kind: Event lastTimestamp: "2026-05-20T14:03:19Z" message: Scaled up replica set isvc-logger-raw-cbb25-predictor-5959584f64 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:19Z" name: isvc-logger-raw-cbb25-predictor.18b14b020257df39 namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: 025d0c23-3865-4203-9ccc-08ac92bd38b2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:03:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-cbb25-predictor namespace: kserve-ci-e2e-test resourceVersion: "16454" uid: f2a4360a-e31b-457e-aee7-db9b44311a4d kind: Event lastTimestamp: "2026-05-20T14:03:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:03:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:49Z" name: isvc-logger-raw-cbb25-predictor.18b14b0580e164ce namespace: kserve-ci-e2e-test resourceVersion: "16739" uid: 4ccb8a4a-35b6-48c8-b5d4-3779fdc58305 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:03:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-cbb25-predictor namespace: kserve-ci-e2e-test resourceVersion: "16454" uid: f2a4360a-e31b-457e-aee7-db9b44311a4d kind: Event lastTimestamp: "2026-05-20T14:03:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:03:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:49Z" name: isvc-logger-raw-cbb25-predictor.18b14b05813eccf6 namespace: kserve-ci-e2e-test resourceVersion: "16740" uid: 8f4e5c03-4092-4ddb-9bdd-4eacc227f7a8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:04:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-cbb25-predictor namespace: kserve-ci-e2e-test resourceVersion: "16629" uid: f2a4360a-e31b-457e-aee7-db9b44311a4d kind: Event lastTimestamp: "2026-05-20T14:04:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:04:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:04:34Z" name: isvc-logger-raw-cbb25-predictor.18b14b0c7eb14bf9 namespace: kserve-ci-e2e-test resourceVersion: "17068" uid: 47378b88-289e-4628-9738-f147b35d907c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:04:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-cbb25-predictor namespace: kserve-ci-e2e-test resourceVersion: "16629" uid: f2a4360a-e31b-457e-aee7-db9b44311a4d kind: Event lastTimestamp: "2026-05-20T14:04:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:04:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:04:34Z" name: isvc-logger-raw-cbb25-predictor.18b14b0c7f13e44a namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: 437232f2-cb7d-4d29-b274-f8d096eff2b5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-05-20T14:03:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-cbb25 namespace: kserve-ci-e2e-test resourceVersion: "16444" uid: 6cc56eef-d935-465a-b132-2a33adcb46e2 kind: Event lastTimestamp: "2026-05-20T14:04:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:04:33Z" name: isvc-logger-raw-cbb25.18b14b02022c87d0 namespace: kserve-ci-e2e-test resourceVersion: "17056" uid: 1373dcd0-cd6e-49a7-8e12-1284809471ca reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:03:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-cbb25 namespace: kserve-ci-e2e-test resourceVersion: "16444" uid: 6cc56eef-d935-465a-b132-2a33adcb46e2 kind: Event lastTimestamp: "2026-05-20T14:03:19Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-cbb25": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-cbb25": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:03:19Z" name: isvc-logger-raw-cbb25.18b14b0202843588 namespace: kserve-ci-e2e-test resourceVersion: "16478" uid: 48223419-16ce-49f5-920a-959d04976c3b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-cbb25 namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: 6cc56eef-d935-465a-b132-2a33adcb46e2 kind: Event lastTimestamp: "2026-05-20T14:03:19Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-cbb25": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:03:19Z" name: isvc-logger-raw-cbb25.18b14b0207ebe299 namespace: kserve-ci-e2e-test resourceVersion: "16479" uid: d060d21f-06f5-4eb0-b9c0-3e99fc175d44 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-cbb25 namespace: kserve-ci-e2e-test resourceVersion: "17053" uid: 6cc56eef-d935-465a-b132-2a33adcb46e2 kind: Event lastTimestamp: "2026-05-20T14:04:32Z" message: InferenceService [isvc-logger-raw-cbb25] is Ready metadata: creationTimestamp: "2026-05-20T14:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:04:32Z" name: isvc-logger-raw-cbb25.18b14b130bf0ce99 namespace: kserve-ci-e2e-test resourceVersion: "17054" uid: 430ecc7e-86b9-42f6-93a1-cabba155acc4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:07:15.342646Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl namespace: kserve-ci-e2e-test resourceVersion: "18360" uid: 57e8517c-0f31-46df-a82f-354bff9abb61 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-cc9515-predictor-79d47ccd86-dm7kl to ip-10-0-140-251.ec2.internal metadata: creationTimestamp: "2026-05-20T14:07:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:07:15Z" name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl.18b14b38da698ba4 namespace: kserve-ci-e2e-test resourceVersion: "18364" uid: ce01a2f2-ac84-436d-a5c6-35a44ef0797f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-c8868bc95-7kctc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:07:15Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl namespace: kserve-ci-e2e-test resourceVersion: "18362" uid: 57e8517c-0f31-46df-a82f-354bff9abb61 kind: Event lastTimestamp: "2026-05-20T14:07:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-primary-cc9515-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:07:15Z" name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl.18b14b38e6810276 namespace: kserve-ci-e2e-test resourceVersion: "18374" uid: ccbaf1fe-ef83-43ec-a69d-9d0923736889 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:07:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl namespace: kserve-ci-e2e-test resourceVersion: "18370" uid: 57e8517c-0f31-46df-a82f-354bff9abb61 kind: Event lastTimestamp: "2026-05-20T14:07:16Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:07:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:07:16Z" name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl.18b14b39180d5b5c namespace: kserve-ci-e2e-test resourceVersion: "18388" uid: 23eb35ea-b7f9-4ebc-bc25-ebf235495bce reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:07:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl namespace: kserve-ci-e2e-test resourceVersion: "18362" uid: 57e8517c-0f31-46df-a82f-354bff9abb61 kind: Event lastTimestamp: "2026-05-20T14:07:16Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0459e31fff147b1f9487117ac00a86221572ce2251d3068d36cf76ed2126ea4f" already present on machine metadata: creationTimestamp: "2026-05-20T14:07:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:07:16Z" name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl.18b14b39198842c0 namespace: kserve-ci-e2e-test resourceVersion: "18391" uid: b901bcbc-d0ce-4e72-9249-dff444fae089 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:07:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl namespace: kserve-ci-e2e-test resourceVersion: "18362" uid: 57e8517c-0f31-46df-a82f-354bff9abb61 kind: Event lastTimestamp: "2026-05-20T14:07:16Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T14:07:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:07:16Z" name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl.18b14b391c7e08e6 namespace: kserve-ci-e2e-test resourceVersion: "18392" uid: 5248df86-4bca-4be8-baa3-f06308cf060a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:07:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl namespace: kserve-ci-e2e-test resourceVersion: "18362" uid: 57e8517c-0f31-46df-a82f-354bff9abb61 kind: Event lastTimestamp: "2026-05-20T14:07:16Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T14:07:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:07:16Z" name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl.18b14b391cf75235 namespace: kserve-ci-e2e-test resourceVersion: "18393" uid: 53994556-e512-4bdf-a5e3-e40a80d241d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:07:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl namespace: kserve-ci-e2e-test resourceVersion: "18362" uid: 57e8517c-0f31-46df-a82f-354bff9abb61 kind: Event lastTimestamp: "2026-05-20T14:07:20Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:07:20Z" name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl.18b14b3a14f5966c namespace: kserve-ci-e2e-test resourceVersion: "18427" uid: 63850760-b131-4c3b-b77c-9b7c67a0c478 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:07:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl namespace: kserve-ci-e2e-test resourceVersion: "18362" uid: 57e8517c-0f31-46df-a82f-354bff9abb61 kind: Event lastTimestamp: "2026-05-20T14:07:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:07:20Z" name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl.18b14b3a182030f3 namespace: kserve-ci-e2e-test resourceVersion: "18429" uid: 8e351120-16ea-4af1-945f-56ee0e2d54af reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:07:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl namespace: kserve-ci-e2e-test resourceVersion: "18362" uid: 57e8517c-0f31-46df-a82f-354bff9abb61 kind: Event lastTimestamp: "2026-05-20T14:07:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:07:20Z" name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl.18b14b3a18a1e56e namespace: kserve-ci-e2e-test resourceVersion: "18432" uid: 40a4ca9e-f159-4632-9729-bc980022f5cb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:07:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl namespace: kserve-ci-e2e-test resourceVersion: "18362" uid: 57e8517c-0f31-46df-a82f-354bff9abb61 kind: Event lastTimestamp: "2026-05-20T14:07:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:07:20Z" name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl.18b14b3a18afb972 namespace: kserve-ci-e2e-test resourceVersion: "18433" uid: 28cff99b-24c9-4a0c-b26b-919977d325c4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:07:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl namespace: kserve-ci-e2e-test resourceVersion: "18362" uid: 57e8517c-0f31-46df-a82f-354bff9abb61 kind: Event lastTimestamp: "2026-05-20T14:07:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:07:20Z" name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl.18b14b3a1c4de55c namespace: kserve-ci-e2e-test resourceVersion: "18435" uid: f294fad2-a9fe-4d9d-9c12-c81c54dc813d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:07:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl namespace: kserve-ci-e2e-test resourceVersion: "18362" uid: 57e8517c-0f31-46df-a82f-354bff9abb61 kind: Event lastTimestamp: "2026-05-20T14:07:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:07:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:07:20Z" name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl.18b14b3a1ccd22ea namespace: kserve-ci-e2e-test resourceVersion: "18436" uid: 860e47ad-a965-4155-817c-e453abdd0072 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-20T14:07:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl namespace: kserve-ci-e2e-test resourceVersion: "18362" uid: 57e8517c-0f31-46df-a82f-354bff9abb61 kind: Event lastTimestamp: "2026-05-20T14:08:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:07:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:08:18Z" name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl.18b14b3a8cbb7c50 namespace: kserve-ci-e2e-test resourceVersion: "18849" uid: eb247a71-4759-4bfb-87ed-74c6965e67d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:08:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl namespace: kserve-ci-e2e-test resourceVersion: "18362" uid: 57e8517c-0f31-46df-a82f-354bff9abb61 kind: Event lastTimestamp: "2026-05-20T14:08:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:08:49Z" name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl.18b14b4ecaac07b8 namespace: kserve-ci-e2e-test resourceVersion: "19182" uid: a0464569-7f8d-4ab7-9fc5-ff10af72c12d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:08:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl namespace: kserve-ci-e2e-test resourceVersion: "18362" uid: 57e8517c-0f31-46df-a82f-354bff9abb61 kind: Event lastTimestamp: "2026-05-20T14:08:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:08:49Z" name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl.18b14b4ecaac861f namespace: kserve-ci-e2e-test resourceVersion: "19185" uid: 12871eec-5a42-41fe-b47d-e54c9d44d16a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:08:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl namespace: kserve-ci-e2e-test resourceVersion: "18362" uid: 57e8517c-0f31-46df-a82f-354bff9abb61 kind: Event lastTimestamp: "2026-05-20T14:08:53Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:08:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:08:53Z" name: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl.18b14b4fbcddf267 namespace: kserve-ci-e2e-test resourceVersion: "19263" uid: 9869d9dd-7073-4c32-acae-843d2dff9fdb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:07:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-cc9515-predictor-79d47ccd86 namespace: kserve-ci-e2e-test resourceVersion: "18350" uid: 996c2f87-874b-4e30-8b9b-381d3835caab kind: Event lastTimestamp: "2026-05-20T14:07:15Z" message: 'Created pod: isvc-primary-cc9515-predictor-79d47ccd86-dm7kl' metadata: creationTimestamp: "2026-05-20T14:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:07:15Z" name: isvc-primary-cc9515-predictor-79d47ccd86.18b14b38da0988d3 namespace: kserve-ci-e2e-test resourceVersion: "18361" uid: 3e4ca232-9767-4d1f-bfb2-42ec0ad6b1a5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:07:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-cc9515-predictor namespace: kserve-ci-e2e-test resourceVersion: "18348" uid: f6af1dd4-cfe0-45a1-b6fa-34be37c85b47 kind: Event lastTimestamp: "2026-05-20T14:07:15Z" message: Scaled up replica set isvc-primary-cc9515-predictor-79d47ccd86 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:07:15Z" name: isvc-primary-cc9515-predictor.18b14b38d5d4285a namespace: kserve-ci-e2e-test resourceVersion: "18353" uid: d5e99659-f266-4478-a978-843597d97187 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:07:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-cc9515-predictor namespace: kserve-ci-e2e-test resourceVersion: "18349" uid: 87f7927c-5260-4b42-826d-4db01b6f960a kind: Event lastTimestamp: "2026-05-20T14:07:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:07:45Z" name: isvc-primary-cc9515-predictor.18b14b3c54f6390a namespace: kserve-ci-e2e-test resourceVersion: "18616" uid: 71959a51-5b1a-47e4-941f-86c074480bb7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:07:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-cc9515-predictor namespace: kserve-ci-e2e-test resourceVersion: "18349" uid: 87f7927c-5260-4b42-826d-4db01b6f960a kind: Event lastTimestamp: "2026-05-20T14:07:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:07:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:07:45Z" name: isvc-primary-cc9515-predictor.18b14b3c55647d17 namespace: kserve-ci-e2e-test resourceVersion: "18617" uid: 7bd81c26-db8e-4b1c-ac4f-7e6b31ca4ec5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:08:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-cc9515-predictor namespace: kserve-ci-e2e-test resourceVersion: "18513" uid: 87f7927c-5260-4b42-826d-4db01b6f960a kind: Event lastTimestamp: "2026-05-20T14:08:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:08:45Z" name: isvc-primary-cc9515-predictor.18b14b43534218f8 namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: f5c50f77-d1ee-4681-88ed-bf85a6843647 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:08:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-cc9515-predictor namespace: kserve-ci-e2e-test resourceVersion: "18513" uid: 87f7927c-5260-4b42-826d-4db01b6f960a kind: Event lastTimestamp: "2026-05-20T14:08:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:08:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:08:45Z" name: isvc-primary-cc9515-predictor.18b14b4353aae811 namespace: kserve-ci-e2e-test resourceVersion: "19102" uid: 55661c8d-1d45-424d-81ff-d4a4a8b93a38 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-05-20T14:07:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-cc9515 namespace: kserve-ci-e2e-test resourceVersion: "18318" uid: 8fa569d5-903c-4d48-942b-f594f2e2acd9 kind: Event lastTimestamp: "2026-05-20T14:08:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:08:28Z" name: isvc-primary-cc9515.18b14b38d5b7f889 namespace: kserve-ci-e2e-test resourceVersion: "18925" uid: dc58433b-7f2b-4434-ae44-0a5d1dffee64 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:07:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-cc9515 namespace: kserve-ci-e2e-test resourceVersion: "18318" uid: 8fa569d5-903c-4d48-942b-f594f2e2acd9 kind: Event lastTimestamp: "2026-05-20T14:07:15Z" message: 'Failed to update status for InferenceService "isvc-primary-cc9515": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-cc9515": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:07:15Z" name: isvc-primary-cc9515.18b14b38d6209ead namespace: kserve-ci-e2e-test resourceVersion: "18355" uid: 825e2b14-4288-4f0d-85a8-b0f2d463e997 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:07:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-cc9515 namespace: kserve-ci-e2e-test resourceVersion: "18318" uid: 8fa569d5-903c-4d48-942b-f594f2e2acd9 kind: Event lastTimestamp: "2026-05-20T14:07:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-cc9515": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:07:15Z" name: isvc-primary-cc9515.18b14b38d620f884 namespace: kserve-ci-e2e-test resourceVersion: "18356" uid: bcd7f182-168f-488b-b7f4-fc9c120310be reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:08:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-cc9515 namespace: kserve-ci-e2e-test resourceVersion: "18923" uid: 8fa569d5-903c-4d48-942b-f594f2e2acd9 kind: Event lastTimestamp: "2026-05-20T14:08:28Z" message: InferenceService [isvc-primary-cc9515] is Ready metadata: creationTimestamp: "2026-05-20T14:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:08:28Z" name: isvc-primary-cc9515.18b14b49f06253cd namespace: kserve-ci-e2e-test resourceVersion: "18924" uid: 86733863-a889-4839-bdd6-676389a6a653 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T13:57:54.153642Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13448" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt to ip-10-0-140-251.ec2.internal metadata: creationTimestamp: "2026-05-20T13:57:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T13:57:54Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14ab630f0e2ac namespace: kserve-ci-e2e-test resourceVersion: "13454" uid: 10ddb2a5-0af6-444a-8447-c0fcd7f6d687 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-c8868bc95-7kctc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:57:54Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:57:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-raw-sklearn-batcher-0beca-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T13:57:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:57:54Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14ab63d08f8c9 namespace: kserve-ci-e2e-test resourceVersion: "13465" uid: 8d162535-3aee-4c8a-af3b-d47d10037dad reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:57:55Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13459" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:57:55Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T13:57:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T13:57:55Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14ab66e6ef252 namespace: kserve-ci-e2e-test resourceVersion: "13483" uid: 89b76afe-cfe2-44bf-90c6-738c9d0f21f1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:57:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:57:55Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0459e31fff147b1f9487117ac00a86221572ce2251d3068d36cf76ed2126ea4f" already present on machine metadata: creationTimestamp: "2026-05-20T13:57:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:57:55Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14ab6701cd9ef namespace: kserve-ci-e2e-test resourceVersion: "13485" uid: 14c03586-15cc-4eff-9ccb-fcc2c556ed1b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:57:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:57:55Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T13:57:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:57:55Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14ab672e28a59 namespace: kserve-ci-e2e-test resourceVersion: "13486" uid: 1189684f-e2aa-4029-ae0e-250059822a79 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:57:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:57:55Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T13:57:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:57:55Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14ab67356c70d namespace: kserve-ci-e2e-test resourceVersion: "13487" uid: 56b99328-fd64-4b77-a85d-6e7a9e393795 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:57:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:57:59Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1501" metadata: creationTimestamp: "2026-05-20T13:57:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:57:59Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14ab77a649b3a namespace: kserve-ci-e2e-test resourceVersion: "13520" uid: fdf60ba2-48ef-467c-ac29-be3d1bd3ff63 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:58:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:58:12Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1501" in 12.477s (12.477s including waiting). Image size: 1560929697 bytes.' metadata: creationTimestamp: "2026-05-20T13:58:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:58:12Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14aba621c48f0 namespace: kserve-ci-e2e-test resourceVersion: "13614" uid: cf20ff8d-19fe-4c12-be50-6581f6ada170 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:58:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:58:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T13:58:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:58:12Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14aba65e5d0bc namespace: kserve-ci-e2e-test resourceVersion: "13616" uid: 941bf2d4-dd95-46ad-98c4-cc6355532b7a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:58:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:58:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T13:58:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:58:12Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14aba665cf778 namespace: kserve-ci-e2e-test resourceVersion: "13617" uid: 117655ec-54f6-45de-8c99-cfa0be671a40 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:58:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:58:12Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-20T13:58:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:58:12Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14aba66694474 namespace: kserve-ci-e2e-test resourceVersion: "13618" uid: 4663649e-12dd-454e-8b9f-f413f32a5ed7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:58:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:58:14Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.313s (2.313s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-20T13:58:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:58:14Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14abaf0555cc2 namespace: kserve-ci-e2e-test resourceVersion: "13633" uid: c2829577-34fc-4bfd-8365-e2f619874d2b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:58:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:58:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T13:58:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:58:14Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14abaf37efe84 namespace: kserve-ci-e2e-test resourceVersion: "13634" uid: f0d2e7f3-5912-40ca-a25d-5a6e058f411b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:58:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:58:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T13:58:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:58:14Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14abaf3f9b480 namespace: kserve-ci-e2e-test resourceVersion: "13636" uid: e9b8801f-b2df-47ce-9842-504fd26f248a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:58:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:58:14Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:093b5585535b7b4890c43b46d80238c68db97d853fe4f3a7840d67abf2e0a4b5" metadata: creationTimestamp: "2026-05-20T13:58:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:58:14Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14abaf4090419 namespace: kserve-ci-e2e-test resourceVersion: "13638" uid: 8a91eda1-0ff3-4bcd-bc1e-1decb451c9b8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:58:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:58:17Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:093b5585535b7b4890c43b46d80238c68db97d853fe4f3a7840d67abf2e0a4b5" in 2.822s (2.822s including waiting). Image size: 239688438 bytes.' metadata: creationTimestamp: "2026-05-20T13:58:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:58:17Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14abb9c3f3170 namespace: kserve-ci-e2e-test resourceVersion: "13659" uid: c364925a-ffda-4f19-9950-a62e9138e7e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:58:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:58:17Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-05-20T13:58:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:58:17Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14abb9ffc8cd5 namespace: kserve-ci-e2e-test resourceVersion: "13660" uid: 0d4567b1-fb4d-4416-8446-96ac5ca93026 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:58:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:58:17Z" message: Started container agent metadata: creationTimestamp: "2026-05-20T13:58:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:58:17Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14abba0744301 namespace: kserve-ci-e2e-test resourceVersion: "13661" uid: a6cb21b0-10f7-4388-909b-14dd893e0f26 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-20T13:58:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:59:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T13:58:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:40Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14abbeb57c5f3 namespace: kserve-ci-e2e-test resourceVersion: "14408" uid: d9824103-9f4e-4e12-995e-a15c043f733b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-20T13:58:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:59:40Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T13:58:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:40Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14abbeb61ccac namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: 1516607b-5733-4734-9bce-c65dec4b5e3a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:59:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T13:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:29Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14acc4f1ff5cc namespace: kserve-ci-e2e-test resourceVersion: "14217" uid: ea1a8a58-294e-4537-8c7c-b84af2f2fc60 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:59:29Z" message: Stopping container agent metadata: creationTimestamp: "2026-05-20T13:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:29Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14acc4f21226e namespace: kserve-ci-e2e-test resourceVersion: "14220" uid: e3533235-724a-4177-972c-3a502a6cf9de reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:59:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T13:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:29Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14acc4f2169df namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: f1cfd047-c937-49ec-ac85-8392045edab2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T13:59:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt namespace: kserve-ci-e2e-test resourceVersion: "13452" uid: 46a9250f-c371-4271-8628-aba0317bb7d1 kind: Event lastTimestamp: "2026-05-20T13:59:44Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T13:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:44Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt.18b14acc73708185 namespace: kserve-ci-e2e-test resourceVersion: "14447" uid: b22e749e-1ff1-41dc-87bc-f2f2de0eb571 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:57:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc namespace: kserve-ci-e2e-test resourceVersion: "13438" uid: 31c1f35a-d146-431b-8828-b71b83d79655 kind: Event lastTimestamp: "2026-05-20T13:57:54Z" message: 'Created pod: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc-zjkzt' metadata: creationTimestamp: "2026-05-20T13:57:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T13:57:54Z" name: isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc.18b14ab6305066b2 namespace: kserve-ci-e2e-test resourceVersion: "13450" uid: d4a6e596-f418-4bf5-b25b-e017d091afcd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:57:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-0beca-predictor namespace: kserve-ci-e2e-test resourceVersion: "13437" uid: a36bace1-b61c-43a1-a81e-586d206cacbd kind: Event lastTimestamp: "2026-05-20T13:57:54Z" message: Scaled up replica set isvc-raw-sklearn-batcher-0beca-predictor-6bdcf7c4dc from 0 to 1 metadata: creationTimestamp: "2026-05-20T13:57:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T13:57:54Z" name: isvc-raw-sklearn-batcher-0beca-predictor.18b14ab6294ed830 namespace: kserve-ci-e2e-test resourceVersion: "13440" uid: 98df6c60-a721-4c36-bea6-1f1bd842a476 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T13:58:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-0beca-predictor namespace: kserve-ci-e2e-test resourceVersion: "13446" uid: d330516e-4d1e-413a-b406-f02cdbf31c82 kind: Event lastTimestamp: "2026-05-20T13:58:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T13:58:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T13:58:54Z" name: isvc-raw-sklearn-batcher-0beca-predictor.18b14ab9af244bf9 namespace: kserve-ci-e2e-test resourceVersion: "13930" uid: 6702e4c4-9c49-4b11-becd-32a8db7ff32f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T13:58:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-0beca-predictor namespace: kserve-ci-e2e-test resourceVersion: "13446" uid: d330516e-4d1e-413a-b406-f02cdbf31c82 kind: Event lastTimestamp: "2026-05-20T13:58:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T13:58:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T13:58:54Z" name: isvc-raw-sklearn-batcher-0beca-predictor.18b14ab9af8b7e18 namespace: kserve-ci-e2e-test resourceVersion: "13931" uid: 8e13d70e-22fc-4c10-912a-f985aebe48db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T13:59:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-0beca-predictor namespace: kserve-ci-e2e-test resourceVersion: "13590" uid: d330516e-4d1e-413a-b406-f02cdbf31c82 kind: Event lastTimestamp: "2026-05-20T13:59:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T13:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T13:59:24Z" name: isvc-raw-sklearn-batcher-0beca-predictor.18b14ac7aba2642b namespace: kserve-ci-e2e-test resourceVersion: "14150" uid: ea01c158-894f-48c0-8cfa-28d72f3c1556 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T13:59:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-0beca-predictor namespace: kserve-ci-e2e-test resourceVersion: "13590" uid: d330516e-4d1e-413a-b406-f02cdbf31c82 kind: Event lastTimestamp: "2026-05-20T13:59:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T13:59:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T13:59:24Z" name: isvc-raw-sklearn-batcher-0beca-predictor.18b14ac7ac0432ae namespace: kserve-ci-e2e-test resourceVersion: "14151" uid: 896df271-7e8e-4712-b6ed-f2a2b82c62fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-05-20T13:57:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-0beca namespace: kserve-ci-e2e-test resourceVersion: "13427" uid: 2b3b8bd7-184e-4fa5-abff-0dd1199249bb kind: Event lastTimestamp: "2026-05-20T13:59:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T13:57:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T13:59:20Z" name: isvc-raw-sklearn-batcher-0beca.18b14ab62f8b1b4d namespace: kserve-ci-e2e-test resourceVersion: "14128" uid: a6afa2e1-d04a-4f3c-b4a0-85e885e4c9a1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T13:57:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-0beca namespace: kserve-ci-e2e-test resourceVersion: "13427" uid: 2b3b8bd7-184e-4fa5-abff-0dd1199249bb kind: Event lastTimestamp: "2026-05-20T13:57:54Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-0beca": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-0beca": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T13:57:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T13:57:54Z" name: isvc-raw-sklearn-batcher-0beca.18b14ab62ff1ef5e namespace: kserve-ci-e2e-test resourceVersion: "13463" uid: 71400305-64d5-4048-bb34-8dd50e032e96 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T13:57:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-0beca namespace: kserve-ci-e2e-test resourceVersion: "13427" uid: 2b3b8bd7-184e-4fa5-abff-0dd1199249bb kind: Event lastTimestamp: "2026-05-20T13:57:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-0beca": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T13:57:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T13:57:54Z" name: isvc-raw-sklearn-batcher-0beca.18b14ab62ff24e05 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: aab05c17-3ad0-4fe5-987f-22fb16828a76 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-0beca namespace: kserve-ci-e2e-test resourceVersion: "14126" uid: 2b3b8bd7-184e-4fa5-abff-0dd1199249bb kind: Event lastTimestamp: "2026-05-20T13:59:20Z" message: InferenceService [isvc-raw-sklearn-batcher-0beca] is Ready metadata: creationTimestamp: "2026-05-20T13:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T13:59:20Z" name: isvc-raw-sklearn-batcher-0beca.18b14aca600ac121 namespace: kserve-ci-e2e-test resourceVersion: "14127" uid: 46489286-7135-4c40-9506-59a610e33892 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:08:35.469230Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-cc9515-predictor-6bfcd87c77-h76gg namespace: kserve-ci-e2e-test resourceVersion: "19001" uid: 70e3ed0e-95e2-4dbc-8968-2abe3a782dae kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-cc9515-predictor-6bfcd87c77-h76gg to ip-10-0-140-251.ec2.internal metadata: creationTimestamp: "2026-05-20T14:08:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:08:35Z" name: isvc-secondary-cc9515-predictor-6bfcd87c77-h76gg.18b14b4b82542811 namespace: kserve-ci-e2e-test resourceVersion: "19005" uid: 46426538-2612-4fe0-8690-a191f13ec63c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-c8868bc95-7kctc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:08:35Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-cc9515-predictor-6bfcd87c77-h76gg namespace: kserve-ci-e2e-test resourceVersion: "19010" uid: 70e3ed0e-95e2-4dbc-8968-2abe3a782dae kind: Event lastTimestamp: "2026-05-20T14:08:35Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:08:35Z" name: isvc-secondary-cc9515-predictor-6bfcd87c77-h76gg.18b14b4b9bbee823 namespace: kserve-ci-e2e-test resourceVersion: "19017" uid: 55d2c8d1-a7a5-406d-b56b-3ef3d2abd89f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:08:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-cc9515-predictor-6bfcd87c77-h76gg namespace: kserve-ci-e2e-test resourceVersion: "19003" uid: 70e3ed0e-95e2-4dbc-8968-2abe3a782dae kind: Event lastTimestamp: "2026-05-20T14:08:38Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0459e31fff147b1f9487117ac00a86221572ce2251d3068d36cf76ed2126ea4f" already present on machine metadata: creationTimestamp: "2026-05-20T14:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:08:38Z" name: isvc-secondary-cc9515-predictor-6bfcd87c77-h76gg.18b14b4b9d2cb1ca namespace: kserve-ci-e2e-test resourceVersion: "19051" uid: e0838680-a6d5-46f7-8d82-e765eef97a08 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:08:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-cc9515-predictor-6bfcd87c77-h76gg namespace: kserve-ci-e2e-test resourceVersion: "19003" uid: 70e3ed0e-95e2-4dbc-8968-2abe3a782dae kind: Event lastTimestamp: "2026-05-20T14:08:38Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T14:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:08:38Z" name: isvc-secondary-cc9515-predictor-6bfcd87c77-h76gg.18b14b4ba0e46cbc namespace: kserve-ci-e2e-test resourceVersion: "19055" uid: 5d73be52-437d-4293-a334-73df5eb085a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:08:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-cc9515-predictor-6bfcd87c77-h76gg namespace: kserve-ci-e2e-test resourceVersion: "19003" uid: 70e3ed0e-95e2-4dbc-8968-2abe3a782dae kind: Event lastTimestamp: "2026-05-20T14:08:38Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T14:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:08:38Z" name: isvc-secondary-cc9515-predictor-6bfcd87c77-h76gg.18b14b4ba164db20 namespace: kserve-ci-e2e-test resourceVersion: "19056" uid: 5db2aab6-2c87-4038-8620-8cd518765aa1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:08:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-cc9515-predictor-6bfcd87c77-h76gg namespace: kserve-ci-e2e-test resourceVersion: "19003" uid: 70e3ed0e-95e2-4dbc-8968-2abe3a782dae kind: Event lastTimestamp: "2026-05-20T14:08:45Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-cc9515-predictor-6bfcd87c77-h76gg_kserve-ci-e2e-test(70e3ed0e-95e2-4dbc-8968-2abe3a782dae) metadata: creationTimestamp: "2026-05-20T14:08:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:08:45Z" name: isvc-secondary-cc9515-predictor-6bfcd87c77-h76gg.18b14b4df100fb0f namespace: kserve-ci-e2e-test resourceVersion: "19109" uid: 52bbbb9b-1eb9-4f6c-a990-4af25f5d5c8d reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:08:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-cc9515-predictor-6bfcd87c77 namespace: kserve-ci-e2e-test resourceVersion: "18991" uid: 7227511e-9e01-4245-b0d7-dfa229d11e14 kind: Event lastTimestamp: "2026-05-20T14:08:35Z" message: 'Created pod: isvc-secondary-cc9515-predictor-6bfcd87c77-h76gg' metadata: creationTimestamp: "2026-05-20T14:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:08:35Z" name: isvc-secondary-cc9515-predictor-6bfcd87c77.18b14b4b81b0db35 namespace: kserve-ci-e2e-test resourceVersion: "19002" uid: 252e4c40-30a4-4739-9e28-1f76a396c4c1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:08:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-cc9515-predictor namespace: kserve-ci-e2e-test resourceVersion: "18989" uid: 9d8bb039-2e33-409d-844c-7b4c605ea5cd kind: Event lastTimestamp: "2026-05-20T14:08:35Z" message: Scaled up replica set isvc-secondary-cc9515-predictor-6bfcd87c77 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:08:35Z" name: isvc-secondary-cc9515-predictor.18b14b4b7ebda51c namespace: kserve-ci-e2e-test resourceVersion: "18994" uid: 58d25744-7c00-4323-9815-f27114197fe9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-05-20T14:08:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-cc9515 namespace: kserve-ci-e2e-test resourceVersion: "18979" uid: 514f6543-a247-4009-8e9c-cdb9352be607 kind: Event lastTimestamp: "2026-05-20T14:08:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:08:46Z" name: isvc-secondary-cc9515.18b14b4b7e91096d namespace: kserve-ci-e2e-test resourceVersion: "19113" uid: db00c2ba-30fb-4405-89d0-4bd20f1e6710 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:08:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-cc9515 namespace: kserve-ci-e2e-test resourceVersion: "18979" uid: 514f6543-a247-4009-8e9c-cdb9352be607 kind: Event lastTimestamp: "2026-05-20T14:08:35Z" message: 'Failed to update status for InferenceService "isvc-secondary-cc9515": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-cc9515": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:08:35Z" name: isvc-secondary-cc9515.18b14b4b7efe89ec namespace: kserve-ci-e2e-test resourceVersion: "18996" uid: 3540ff51-a958-4676-9ecd-89d2daf674cd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:08:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-cc9515 namespace: kserve-ci-e2e-test resourceVersion: "18979" uid: 514f6543-a247-4009-8e9c-cdb9352be607 kind: Event lastTimestamp: "2026-05-20T14:08:35Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-cc9515": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:08:35Z" name: isvc-secondary-cc9515.18b14b4b7eff37a7 namespace: kserve-ci-e2e-test resourceVersion: "18997" uid: bbd76f6b-5ed3-4603-b785-1f23f3bd5dd1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-20T13:59:29.255277Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd namespace: kserve-ci-e2e-test resourceVersion: "14242" uid: 29a375df-d7c7-4e26-8780-ee6ff78e9dcc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd to ip-10-0-140-251.ec2.internal metadata: creationTimestamp: "2026-05-20T13:59:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T13:59:29Z" name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd.18b14acc5570ae11 namespace: kserve-ci-e2e-test resourceVersion: "14250" uid: edfb4fd4-5a44-43a9-834f-28838c190bd1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-c8868bc95-7kctc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:29Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: 29a375df-d7c7-4e26-8780-ee6ff78e9dcc kind: Event lastTimestamp: "2026-05-20T13:59:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-4de28-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T13:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:29Z" name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd.18b14acc65ba6531 namespace: kserve-ci-e2e-test resourceVersion: "14280" uid: 4a1d3d38-51fe-4e5c-a865-64e6e00a24ed reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:30Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 29a375df-d7c7-4e26-8780-ee6ff78e9dcc kind: Event lastTimestamp: "2026-05-20T13:59:30Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T13:59:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T13:59:30Z" name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd.18b14acc934f380c namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: 930ea624-aa3f-40d9-af8b-c37bba9247eb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: 29a375df-d7c7-4e26-8780-ee6ff78e9dcc kind: Event lastTimestamp: "2026-05-20T13:59:30Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0459e31fff147b1f9487117ac00a86221572ce2251d3068d36cf76ed2126ea4f" already present on machine metadata: creationTimestamp: "2026-05-20T13:59:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:30Z" name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd.18b14acc94d43dac namespace: kserve-ci-e2e-test resourceVersion: "14302" uid: 4965751b-c0c9-4abe-b92a-62ff96e5acba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: 29a375df-d7c7-4e26-8780-ee6ff78e9dcc kind: Event lastTimestamp: "2026-05-20T13:59:30Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T13:59:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:30Z" name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd.18b14acc98140069 namespace: kserve-ci-e2e-test resourceVersion: "14304" uid: 4560d1b7-7310-447e-ba05-729822b76fdf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: 29a375df-d7c7-4e26-8780-ee6ff78e9dcc kind: Event lastTimestamp: "2026-05-20T13:59:30Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T13:59:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:30Z" name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd.18b14acc9896be6b namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: 30bc1905-cfc8-4b9f-9631-84fe6430b257 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: 29a375df-d7c7-4e26-8780-ee6ff78e9dcc kind: Event lastTimestamp: "2026-05-20T13:59:34Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T13:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:34Z" name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd.18b14acd70a6e6c1 namespace: kserve-ci-e2e-test resourceVersion: "14344" uid: d20424ee-7ee5-49ac-8551-1a94d91e691a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: 29a375df-d7c7-4e26-8780-ee6ff78e9dcc kind: Event lastTimestamp: "2026-05-20T13:59:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T13:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:34Z" name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd.18b14acd73e99bc8 namespace: kserve-ci-e2e-test resourceVersion: "14350" uid: 7dae0e9e-6d53-4ab5-ae83-fd7a14846cc2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: 29a375df-d7c7-4e26-8780-ee6ff78e9dcc kind: Event lastTimestamp: "2026-05-20T13:59:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T13:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:34Z" name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd.18b14acd746199a3 namespace: kserve-ci-e2e-test resourceVersion: "14351" uid: 7ae87529-c871-4e63-8cde-15767bdb686c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: 29a375df-d7c7-4e26-8780-ee6ff78e9dcc kind: Event lastTimestamp: "2026-05-20T13:59:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T13:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:34Z" name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd.18b14acd746da5c6 namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 1ed3b901-7a88-47dd-8b54-8d9c24e69154 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: 29a375df-d7c7-4e26-8780-ee6ff78e9dcc kind: Event lastTimestamp: "2026-05-20T13:59:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T13:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:34Z" name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd.18b14acd77ce59d0 namespace: kserve-ci-e2e-test resourceVersion: "14355" uid: 2841a4de-c317-431e-b807-bfc32363ef34 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: 29a375df-d7c7-4e26-8780-ee6ff78e9dcc kind: Event lastTimestamp: "2026-05-20T13:59:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T13:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:34Z" name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd.18b14acd784adb83 namespace: kserve-ci-e2e-test resourceVersion: "14356" uid: 2be82097-118a-45b4-8407-6890825b2548 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-20T13:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: 29a375df-d7c7-4e26-8780-ee6ff78e9dcc kind: Event lastTimestamp: "2026-05-20T14:01:21Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T13:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:21Z" name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd.18b14acdace8baf3 namespace: kserve-ci-e2e-test resourceVersion: "15354" uid: 8f5fe8af-ddc7-45db-9cdb-c1f296464364 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: 29a375df-d7c7-4e26-8780-ee6ff78e9dcc kind: Event lastTimestamp: "2026-05-20T14:01:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:01:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:19Z" name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd.18b14ae60074bf5b namespace: kserve-ci-e2e-test resourceVersion: "15265" uid: 326be941-7464-40ea-8245-45fc040881a3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: 29a375df-d7c7-4e26-8780-ee6ff78e9dcc kind: Event lastTimestamp: "2026-05-20T14:01:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:01:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:19Z" name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd.18b14ae600754112 namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: a3e19ee9-3e65-4055-8344-8612debf2f31 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: 29a375df-d7c7-4e26-8780-ee6ff78e9dcc kind: Event lastTimestamp: "2026-05-20T14:01:21Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:21Z" name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd.18b14ae65b2f5f0b namespace: kserve-ci-e2e-test resourceVersion: "15353" uid: 63a94ad1-35fa-407e-b7ed-af4dc7d15027 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7 namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: ece4a607-67b0-4b69-90f9-a2260f170876 kind: Event lastTimestamp: "2026-05-20T13:59:29Z" message: 'Created pod: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7-cshkd' metadata: creationTimestamp: "2026-05-20T13:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T13:59:29Z" name: isvc-sklearn-graph-raw-4de28-predictor-8754c99b7.18b14acc54f9844d namespace: kserve-ci-e2e-test resourceVersion: "14244" uid: 309e11d1-f92f-42b7-bb4d-4d2ca52d57bf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-4de28-predictor namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: f11fc87c-9da1-49dc-8727-f5bb85e7a20a kind: Event lastTimestamp: "2026-05-20T13:59:29Z" message: Scaled up replica set isvc-sklearn-graph-raw-4de28-predictor-8754c99b7 from 0 to 1 metadata: creationTimestamp: "2026-05-20T13:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T13:59:29Z" name: isvc-sklearn-graph-raw-4de28-predictor.18b14acc507b1bb5 namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: 4e07d734-4c50-490f-bb8e-14f2958a7897 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T13:59:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-4de28-predictor namespace: kserve-ci-e2e-test resourceVersion: "14223" uid: a7b447d4-ca88-4cc0-885a-1a48cf9803d2 kind: Event lastTimestamp: "2026-05-20T14:00:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T13:59:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:00:14Z" name: isvc-sklearn-graph-raw-4de28-predictor.18b14acfcf0f278f namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: 6ec4d240-3904-4e50-8e04-f771e57bb363 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T13:59:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-4de28-predictor namespace: kserve-ci-e2e-test resourceVersion: "14223" uid: a7b447d4-ca88-4cc0-885a-1a48cf9803d2 kind: Event lastTimestamp: "2026-05-20T14:00:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T13:59:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:00:14Z" name: isvc-sklearn-graph-raw-4de28-predictor.18b14acfcf737257 namespace: kserve-ci-e2e-test resourceVersion: "14675" uid: 26aa2eb0-bb6b-4766-a0c7-0c56016998ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T13:59:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-4de28-predictor namespace: kserve-ci-e2e-test resourceVersion: "14435" uid: a7b447d4-ca88-4cc0-885a-1a48cf9803d2 kind: Event lastTimestamp: "2026-05-20T14:01:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T13:59:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:14Z" name: isvc-sklearn-graph-raw-4de28-predictor.18b14ad34f094f9e namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 5fd29137-b6af-4d0f-83d7-9a4b1dbbc86d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T13:59:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-4de28-predictor namespace: kserve-ci-e2e-test resourceVersion: "14435" uid: a7b447d4-ca88-4cc0-885a-1a48cf9803d2 kind: Event lastTimestamp: "2026-05-20T14:01:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T13:59:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:14Z" name: isvc-sklearn-graph-raw-4de28-predictor.18b14ad34f9b07ed namespace: kserve-ci-e2e-test resourceVersion: "15164" uid: d18d2cad-c5d5-412c-9942-72a0d3af2bf0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-20T13:59:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-4de28 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 57a21a18-e414-4769-a777-d18b45be6c91 kind: Event lastTimestamp: "2026-05-20T14:00:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T13:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:00:51Z" name: isvc-sklearn-graph-raw-4de28.18b14acc5057c417 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: ed3982f7-f697-40af-b3b7-07626e95d76c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-4de28 namespace: kserve-ci-e2e-test resourceVersion: "14193" uid: 57a21a18-e414-4769-a777-d18b45be6c91 kind: Event lastTimestamp: "2026-05-20T13:59:29Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-4de28": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-4de28": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T13:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T13:59:29Z" name: isvc-sklearn-graph-raw-4de28.18b14acc50cbd118 namespace: kserve-ci-e2e-test resourceVersion: "14229" uid: 9982cb32-46e5-4341-8a8e-ac270dc3dc01 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:00:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-4de28 namespace: kserve-ci-e2e-test resourceVersion: "14949" uid: 57a21a18-e414-4769-a777-d18b45be6c91 kind: Event lastTimestamp: "2026-05-20T14:00:51Z" message: InferenceService [isvc-sklearn-graph-raw-4de28] is Ready metadata: creationTimestamp: "2026-05-20T14:00:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:00:51Z" name: isvc-sklearn-graph-raw-4de28.18b14adf6636b5b6 namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: c909687a-67d5-4daf-bdd1-dc8455d1c3b2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:01:19.564210Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd namespace: kserve-ci-e2e-test resourceVersion: "15278" uid: 2a73893f-48bb-4696-a9d7-8a63663df93d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd to ip-10-0-140-251.ec2.internal metadata: creationTimestamp: "2026-05-20T14:01:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:01:19Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd.18b14ae6045d808f namespace: kserve-ci-e2e-test resourceVersion: "15286" uid: ee1fc89b-2574-4d97-9399-67f4536fe00c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-c8868bc95-7kctc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:19Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: 2a73893f-48bb-4696-a9d7-8a63663df93d kind: Event lastTimestamp: "2026-05-20T14:01:19Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-hpa-de018-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:01:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:19Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd.18b14ae60d5b302c namespace: kserve-ci-e2e-test resourceVersion: "15319" uid: 4b2458de-6b3d-46cb-bec7-eaf1cc4fa5b3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd namespace: kserve-ci-e2e-test resourceVersion: "15295" uid: 2a73893f-48bb-4696-a9d7-8a63663df93d kind: Event lastTimestamp: "2026-05-20T14:01:20Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:01:20Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd.18b14ae6425119b2 namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: 1a6e2ad2-fcfb-437f-ada1-cd1a04a6d314 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: 2a73893f-48bb-4696-a9d7-8a63663df93d kind: Event lastTimestamp: "2026-05-20T14:01:20Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0459e31fff147b1f9487117ac00a86221572ce2251d3068d36cf76ed2126ea4f" already present on machine metadata: creationTimestamp: "2026-05-20T14:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:20Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd.18b14ae643d2bac9 namespace: kserve-ci-e2e-test resourceVersion: "15347" uid: 577411bd-1c67-46a5-b052-c89a3f3a8e5b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: 2a73893f-48bb-4696-a9d7-8a63663df93d kind: Event lastTimestamp: "2026-05-20T14:01:20Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T14:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:20Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd.18b14ae6470586e4 namespace: kserve-ci-e2e-test resourceVersion: "15348" uid: 4fef6b67-8e16-425d-9da1-500a0c76e8cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: 2a73893f-48bb-4696-a9d7-8a63663df93d kind: Event lastTimestamp: "2026-05-20T14:01:20Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T14:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:20Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd.18b14ae647704923 namespace: kserve-ci-e2e-test resourceVersion: "15349" uid: 474e00b4-32df-438f-baec-5c0c884f2761 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: 2a73893f-48bb-4696-a9d7-8a63663df93d kind: Event lastTimestamp: "2026-05-20T14:01:25Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:25Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd.18b14ae76104c144 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 963572d7-7239-444a-8025-6a10d9899f28 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: 2a73893f-48bb-4696-a9d7-8a63663df93d kind: Event lastTimestamp: "2026-05-20T14:01:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:25Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd.18b14ae764482798 namespace: kserve-ci-e2e-test resourceVersion: "15423" uid: 79bf1f16-4b14-441a-8922-0e6b7b82a5b9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: 2a73893f-48bb-4696-a9d7-8a63663df93d kind: Event lastTimestamp: "2026-05-20T14:01:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:25Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd.18b14ae764d3f7eb namespace: kserve-ci-e2e-test resourceVersion: "15424" uid: 82ae62ec-e9e0-4427-8ad2-6cc7516cb975 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: 2a73893f-48bb-4696-a9d7-8a63663df93d kind: Event lastTimestamp: "2026-05-20T14:01:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:25Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd.18b14ae764dfef02 namespace: kserve-ci-e2e-test resourceVersion: "15426" uid: 9ee02e34-1e02-44be-b2d2-e2eeb89f3240 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: 2a73893f-48bb-4696-a9d7-8a63663df93d kind: Event lastTimestamp: "2026-05-20T14:01:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:25Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd.18b14ae76835c89c namespace: kserve-ci-e2e-test resourceVersion: "15430" uid: 34bb9da5-9682-48a0-ba5f-e4e4895ed185 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: 2a73893f-48bb-4696-a9d7-8a63663df93d kind: Event lastTimestamp: "2026-05-20T14:01:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:25Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd.18b14ae768af6d2f namespace: kserve-ci-e2e-test resourceVersion: "15431" uid: 66fee68c-8851-4fbd-b8c9-089c9946b730 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T14:01:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: 2a73893f-48bb-4696-a9d7-8a63663df93d kind: Event lastTimestamp: "2026-05-20T14:03:02Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:02Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd.18b14ae79d0bef4d namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: b87739b1-47ac-4cce-8628-3f0cd7f95e4f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: 2a73893f-48bb-4696-a9d7-8a63663df93d kind: Event lastTimestamp: "2026-05-20T14:02:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:02:59Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd.18b14afd596dfc64 namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: 1fdd4e76-b606-49e6-be92-c488dbc5d656 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: 2a73893f-48bb-4696-a9d7-8a63663df93d kind: Event lastTimestamp: "2026-05-20T14:02:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:02:59Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd.18b14afd596e6e75 namespace: kserve-ci-e2e-test resourceVersion: "16231" uid: 9cd1da32-d933-47df-8c40-19d47f032463 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: 2a73893f-48bb-4696-a9d7-8a63663df93d kind: Event lastTimestamp: "2026-05-20T14:03:02Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:03:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:02Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd.18b14afdf749f0f2 namespace: kserve-ci-e2e-test resourceVersion: "16298" uid: 0d8580e0-eedc-45d4-8c70-3692361c390c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5 namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: 9830a599-0d67-49ff-9e6f-eaf00d1fb942 kind: Event lastTimestamp: "2026-05-20T14:01:19Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5-tvgmd' metadata: creationTimestamp: "2026-05-20T14:01:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:19Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5.18b14ae603f019e1 namespace: kserve-ci-e2e-test resourceVersion: "15281" uid: ffac2553-7f8e-42e2-a5f4-9ff7c98d0fb2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-de018-predictor namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: 8c384134-29c6-47fb-a7fe-cceb4fcd9ea9 kind: Event lastTimestamp: "2026-05-20T14:01:19Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-de018-predictor-5fd94fb6f5 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:01:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:19Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor.18b14ae5ff4a3c5e namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: 6c029216-e130-4a44-b7cc-b595fdb8ab4e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-de018-predictor namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: 24a416d7-027a-488e-a4d5-ace03e18f104 kind: Event lastTimestamp: "2026-05-20T14:01:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:01:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:34Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor.18b14ae97e4e03dd namespace: kserve-ci-e2e-test resourceVersion: "15499" uid: d7553c78-4b5d-4487-b727-b66c797edd41 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-de018-predictor namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: 24a416d7-027a-488e-a4d5-ace03e18f104 kind: Event lastTimestamp: "2026-05-20T14:01:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:01:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:34Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor.18b14ae97edf0a14 namespace: kserve-ci-e2e-test resourceVersion: "15501" uid: d6ffb5e1-53fa-4762-b51e-a21eaca4c812 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:01:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-de018-predictor namespace: kserve-ci-e2e-test resourceVersion: "15500" uid: 24a416d7-027a-488e-a4d5-ace03e18f104 kind: Event lastTimestamp: "2026-05-20T14:02:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:02:49Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor.18b14aecfd9e0d1d namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 21e93869-d811-431c-863f-8f0541c13a02 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-20T14:01:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-de018-predictor namespace: kserve-ci-e2e-test resourceVersion: "15500" uid: 24a416d7-027a-488e-a4d5-ace03e18f104 kind: Event lastTimestamp: "2026-05-20T14:02:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:02:49Z" name: isvc-sklearn-graph-raw-hpa-de018-predictor.18b14aecfe1091c0 namespace: kserve-ci-e2e-test resourceVersion: "16088" uid: 71d3a913-1245-497f-a9f3-fa2a6033874a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T14:01:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-de018 namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: 61cb19a0-13ba-4fc9-8b0d-b5b851ec50c5 kind: Event lastTimestamp: "2026-05-20T14:02:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:01:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:02:32Z" name: isvc-sklearn-graph-raw-hpa-de018.18b14ae5ff17948a namespace: kserve-ci-e2e-test resourceVersion: "15943" uid: 21702833-8f16-4d4b-90d9-88c4dbe20aa4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-de018 namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: 61cb19a0-13ba-4fc9-8b0d-b5b851ec50c5 kind: Event lastTimestamp: "2026-05-20T14:01:19Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-de018": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-de018": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:01:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:01:19Z" name: isvc-sklearn-graph-raw-hpa-de018.18b14ae5ff83a5ed namespace: kserve-ci-e2e-test resourceVersion: "15260" uid: 9afc2651-6943-4ccc-a965-202820d031ef reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-de018 namespace: kserve-ci-e2e-test resourceVersion: "15941" uid: 61cb19a0-13ba-4fc9-8b0d-b5b851ec50c5 kind: Event lastTimestamp: "2026-05-20T14:02:32Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-de018] is Ready metadata: creationTimestamp: "2026-05-20T14:02:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:02:32Z" name: isvc-sklearn-graph-raw-hpa-de018.18b14af70040fbec namespace: kserve-ci-e2e-test resourceVersion: "15942" uid: f9f00e59-a99c-416b-87bc-0ffd55645522 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:04:45.093073Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62 namespace: kserve-ci-e2e-test resourceVersion: "17184" uid: cd8b523f-a553-40ea-9307-7295088932f0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62 to ip-10-0-140-251.ec2.internal metadata: creationTimestamp: "2026-05-20T14:04:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:04:45Z" name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62.18b14b15ded6f5fb namespace: kserve-ci-e2e-test resourceVersion: "17188" uid: 5354dbe1-daef-49f4-a176-2f57a4c622fb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-c8868bc95-7kctc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:45Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62 namespace: kserve-ci-e2e-test resourceVersion: "17198" uid: cd8b523f-a553-40ea-9307-7295088932f0 kind: Event lastTimestamp: "2026-05-20T14:04:45Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:04:45Z" name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62.18b14b15f874d297 namespace: kserve-ci-e2e-test resourceVersion: "17216" uid: 619540f2-2364-48a8-8e0f-da3f7a3e90f1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62 namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: cd8b523f-a553-40ea-9307-7295088932f0 kind: Event lastTimestamp: "2026-05-20T14:04:45Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0459e31fff147b1f9487117ac00a86221572ce2251d3068d36cf76ed2126ea4f" already present on machine metadata: creationTimestamp: "2026-05-20T14:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:45Z" name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62.18b14b15f9c6458f namespace: kserve-ci-e2e-test resourceVersion: "17218" uid: 080bc98f-e0bb-4718-9212-5828aa82eb4f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62 namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: cd8b523f-a553-40ea-9307-7295088932f0 kind: Event lastTimestamp: "2026-05-20T14:04:45Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T14:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:45Z" name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62.18b14b15fc945fef namespace: kserve-ci-e2e-test resourceVersion: "17219" uid: 85ef0e22-9f8c-415f-a902-327035311b81 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:45Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62 namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: cd8b523f-a553-40ea-9307-7295088932f0 kind: Event lastTimestamp: "2026-05-20T14:04:45Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T14:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:45Z" name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62.18b14b15fd083c45 namespace: kserve-ci-e2e-test resourceVersion: "17220" uid: 66722fd6-be84-4a3c-9f43-aaf7e80002b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62 namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: cd8b523f-a553-40ea-9307-7295088932f0 kind: Event lastTimestamp: "2026-05-20T14:04:50Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:50Z" name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62.18b14b170a699e05 namespace: kserve-ci-e2e-test resourceVersion: "17263" uid: 5a6e012f-fde4-4748-ae5f-b8c7523396a1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62 namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: cd8b523f-a553-40ea-9307-7295088932f0 kind: Event lastTimestamp: "2026-05-20T14:04:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:50Z" name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62.18b14b170d76848f namespace: kserve-ci-e2e-test resourceVersion: "17268" uid: 43bc6520-d1ef-4352-a0b3-125d61a413d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62 namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: cd8b523f-a553-40ea-9307-7295088932f0 kind: Event lastTimestamp: "2026-05-20T14:04:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:50Z" name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62.18b14b170deabc48 namespace: kserve-ci-e2e-test resourceVersion: "17269" uid: 92995d9d-6079-4d6b-8514-743d74a80614 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62 namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: cd8b523f-a553-40ea-9307-7295088932f0 kind: Event lastTimestamp: "2026-05-20T14:04:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:50Z" name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62.18b14b170df67324 namespace: kserve-ci-e2e-test resourceVersion: "17270" uid: f5a2b069-c2a6-4ac3-b579-7aef335a9276 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62 namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: cd8b523f-a553-40ea-9307-7295088932f0 kind: Event lastTimestamp: "2026-05-20T14:04:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:50Z" name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62.18b14b1711458258 namespace: kserve-ci-e2e-test resourceVersion: "17273" uid: a599d4b8-8370-45eb-a961-b667022a946e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62 namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: cd8b523f-a553-40ea-9307-7295088932f0 kind: Event lastTimestamp: "2026-05-20T14:04:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:04:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:50Z" name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62.18b14b1711be7052 namespace: kserve-ci-e2e-test resourceVersion: "17274" uid: 8fe2aff5-7318-42fc-84cb-10437de9452a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 17 eventTime: null firstTimestamp: "2026-05-20T14:04:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62 namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: cd8b523f-a553-40ea-9307-7295088932f0 kind: Event lastTimestamp: "2026-05-20T14:07:20Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:04:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:07:20Z" name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62.18b14b17822820b3 namespace: kserve-ci-e2e-test resourceVersion: "18437" uid: a67f4f92-bab3-4a25-94fa-f527dbc9c91a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:07:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62 namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: cd8b523f-a553-40ea-9307-7295088932f0 kind: Event lastTimestamp: "2026-05-20T14:07:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:07:15Z" name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62.18b14b38d47bc3a8 namespace: kserve-ci-e2e-test resourceVersion: "18343" uid: 6c7507ce-0cb5-45ce-a2a4-1deb36209e52 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:07:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62 namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: cd8b523f-a553-40ea-9307-7295088932f0 kind: Event lastTimestamp: "2026-05-20T14:07:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:07:15Z" name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62.18b14b38d47c4dd9 namespace: kserve-ci-e2e-test resourceVersion: "18345" uid: 34cd6384-c3d9-4e61-8b5b-61df7f49cc07 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:07:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62 namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: cd8b523f-a553-40ea-9307-7295088932f0 kind: Event lastTimestamp: "2026-05-20T14:07:23Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:07:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:07:23Z" name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62.18b14b398093e4c6 namespace: kserve-ci-e2e-test resourceVersion: "18457" uid: a10b1f70-e288-4b50-bb20-8b4a19aae368 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5 namespace: kserve-ci-e2e-test resourceVersion: "17166" uid: a571690a-6a72-45fa-973f-1fb88b115ccd kind: Event lastTimestamp: "2026-05-20T14:04:45Z" message: 'Created pod: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5-2cj62' metadata: creationTimestamp: "2026-05-20T14:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:04:45Z" name: isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5.18b14b15de6a7328 namespace: kserve-ci-e2e-test resourceVersion: "17185" uid: fa7e1432-8a6a-483f-a7e5-7ad7ebc4d7fd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-3983e-predictor namespace: kserve-ci-e2e-test resourceVersion: "17158" uid: 8bf253a4-2a33-4d2f-be1e-b3e3b42b1715 kind: Event lastTimestamp: "2026-05-20T14:04:45Z" message: Scaled up replica set isvc-sklearn-scale-raw-3983e-predictor-6d6c6bcdd5 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:04:45Z" name: isvc-sklearn-scale-raw-3983e-predictor.18b14b15daef3d18 namespace: kserve-ci-e2e-test resourceVersion: "17167" uid: 09aeaa98-4e87-4103-9e2b-e05096f21fd2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:05:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-3983e-predictor namespace: kserve-ci-e2e-test resourceVersion: "17164" uid: 094db4ea-62b0-4c36-b88c-67859d4e96b1 kind: Event lastTimestamp: "2026-05-20T14:05:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:05:00Z" name: isvc-sklearn-scale-raw-3983e-predictor.18b14b1959ece6bb namespace: kserve-ci-e2e-test resourceVersion: "17353" uid: ce809325-b765-4a04-8369-8b845a5b0276 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:05:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-3983e-predictor namespace: kserve-ci-e2e-test resourceVersion: "17164" uid: 094db4ea-62b0-4c36-b88c-67859d4e96b1 kind: Event lastTimestamp: "2026-05-20T14:05:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:05:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:05:00Z" name: isvc-sklearn-scale-raw-3983e-predictor.18b14b195a4af1dc namespace: kserve-ci-e2e-test resourceVersion: "17355" uid: 0eafe950-fde9-4167-802a-66c27cb420fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T14:05:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-3983e-predictor namespace: kserve-ci-e2e-test resourceVersion: "17354" uid: 094db4ea-62b0-4c36-b88c-67859d4e96b1 kind: Event lastTimestamp: "2026-05-20T14:07:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:05:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:07:15Z" name: isvc-sklearn-scale-raw-3983e-predictor.18b14b1cd8fa7b66 namespace: kserve-ci-e2e-test resourceVersion: "18320" uid: a7884699-6d9c-4c40-93b9-6bf378e82fa7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T14:05:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-3983e-predictor namespace: kserve-ci-e2e-test resourceVersion: "17354" uid: 094db4ea-62b0-4c36-b88c-67859d4e96b1 kind: Event lastTimestamp: "2026-05-20T14:07:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:05:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:07:15Z" name: isvc-sklearn-scale-raw-3983e-predictor.18b14b1cd95fab35 namespace: kserve-ci-e2e-test resourceVersion: "18321" uid: 7650faea-178c-4329-9558-a2398ae1c5e5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-05-20T14:04:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-3983e namespace: kserve-ci-e2e-test resourceVersion: "17142" uid: 8cd0f740-6828-472c-adb6-5c83ec99d787 kind: Event lastTimestamp: "2026-05-20T14:07:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:07:11Z" name: isvc-sklearn-scale-raw-3983e.18b14b15daf1368c namespace: kserve-ci-e2e-test resourceVersion: "18286" uid: 748973a4-137a-4b42-bf64-b306e8a98da6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-3983e namespace: kserve-ci-e2e-test resourceVersion: "17142" uid: 8cd0f740-6828-472c-adb6-5c83ec99d787 kind: Event lastTimestamp: "2026-05-20T14:04:45Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-3983e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-3983e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:04:45Z" name: isvc-sklearn-scale-raw-3983e.18b14b15db77ecb1 namespace: kserve-ci-e2e-test resourceVersion: "17174" uid: a5108252-88cd-4b02-9668-e2b0f5b5b16a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:07:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-3983e namespace: kserve-ci-e2e-test resourceVersion: "18283" uid: 8cd0f740-6828-472c-adb6-5c83ec99d787 kind: Event lastTimestamp: "2026-05-20T14:07:11Z" message: InferenceService [isvc-sklearn-scale-raw-3983e] is Ready metadata: creationTimestamp: "2026-05-20T14:07:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:07:11Z" name: isvc-sklearn-scale-raw-3983e.18b14b37d7e516db namespace: kserve-ci-e2e-test resourceVersion: "18285" uid: 5a3de160-c303-463a-b556-66ccbb1fe9ff reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T13:59:29.330187Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn namespace: kserve-ci-e2e-test resourceVersion: "14263" uid: 095473d0-4ee6-42f5-bcf0-ff52bb8660ef kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn to ip-10-0-140-251.ec2.internal metadata: creationTimestamp: "2026-05-20T13:59:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T13:59:29Z" name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn.18b14acc59e7bb72 namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: 998f2af5-2a47-4631-a2e1-a0d5a5461c3c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-c8868bc95-7kctc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:29Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: 095473d0-4ee6-42f5-bcf0-ff52bb8660ef kind: Event lastTimestamp: "2026-05-20T13:59:29Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T13:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T13:59:29Z" name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn.18b14acc73c2c211 namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: 446fa98a-c7f6-451b-80eb-ca53c51e6ab7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 095473d0-4ee6-42f5-bcf0-ff52bb8660ef kind: Event lastTimestamp: "2026-05-20T13:59:29Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0459e31fff147b1f9487117ac00a86221572ce2251d3068d36cf76ed2126ea4f" already present on machine metadata: creationTimestamp: "2026-05-20T13:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:29Z" name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn.18b14acc75445dd2 namespace: kserve-ci-e2e-test resourceVersion: "14288" uid: 52f63f41-4ec1-4c1b-ab35-bfcef82b3986 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 095473d0-4ee6-42f5-bcf0-ff52bb8660ef kind: Event lastTimestamp: "2026-05-20T13:59:29Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T13:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:29Z" name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn.18b14acc7825ff20 namespace: kserve-ci-e2e-test resourceVersion: "14291" uid: b8857db3-5cc3-4d17-82dd-0bd962c02fc7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 095473d0-4ee6-42f5-bcf0-ff52bb8660ef kind: Event lastTimestamp: "2026-05-20T13:59:29Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T13:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:29Z" name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn.18b14acc789f8a6e namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: 28ebaa7b-11dc-4596-a934-fc538bb32262 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 095473d0-4ee6-42f5-bcf0-ff52bb8660ef kind: Event lastTimestamp: "2026-05-20T13:59:34Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-05-20T13:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:34Z" name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn.18b14acd70dbac2e namespace: kserve-ci-e2e-test resourceVersion: "14345" uid: fa497bcd-e3d7-435a-9104-dfae1adfe5b5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 095473d0-4ee6-42f5-bcf0-ff52bb8660ef kind: Event lastTimestamp: "2026-05-20T13:59:54Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 20.477s (20.477s including waiting). Image size: 1306436347 bytes.' metadata: creationTimestamp: "2026-05-20T13:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:54Z" name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn.18b14ad235640430 namespace: kserve-ci-e2e-test resourceVersion: "14520" uid: dea2c405-78f3-49d4-bc97-f0f5648e970d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 095473d0-4ee6-42f5-bcf0-ff52bb8660ef kind: Event lastTimestamp: "2026-05-20T13:59:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T13:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:54Z" name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn.18b14ad238b8ce56 namespace: kserve-ci-e2e-test resourceVersion: "14521" uid: 852516d2-1089-4f15-abcc-7950790ef5b5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 095473d0-4ee6-42f5-bcf0-ff52bb8660ef kind: Event lastTimestamp: "2026-05-20T13:59:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T13:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:54Z" name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn.18b14ad239265504 namespace: kserve-ci-e2e-test resourceVersion: "14522" uid: ff935e0f-e4e1-4a5d-9c64-35c54e41981c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 095473d0-4ee6-42f5-bcf0-ff52bb8660ef kind: Event lastTimestamp: "2026-05-20T13:59:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T13:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:54Z" name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn.18b14ad23932a7a4 namespace: kserve-ci-e2e-test resourceVersion: "14523" uid: 67fe349e-6af8-4d19-8062-26d55c8958ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 095473d0-4ee6-42f5-bcf0-ff52bb8660ef kind: Event lastTimestamp: "2026-05-20T13:59:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T13:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:54Z" name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn.18b14ad23bfa7d6d namespace: kserve-ci-e2e-test resourceVersion: "14524" uid: 3601152e-c565-464d-aead-e31ef0283542 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 095473d0-4ee6-42f5-bcf0-ff52bb8660ef kind: Event lastTimestamp: "2026-05-20T13:59:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T13:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T13:59:54Z" name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn.18b14ad23c6732e2 namespace: kserve-ci-e2e-test resourceVersion: "14526" uid: 389449e0-2558-4e7b-9133-b71d3b027f33 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T13:59:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 095473d0-4ee6-42f5-bcf0-ff52bb8660ef kind: Event lastTimestamp: "2026-05-20T14:01:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T13:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:22Z" name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn.18b14ad29561c0ec namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: 93b90c30-318b-434a-914d-27e47531ac1c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 095473d0-4ee6-42f5-bcf0-ff52bb8660ef kind: Event lastTimestamp: "2026-05-20T14:01:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:01:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:19Z" name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn.18b14ae60a313a52 namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 5c6d4e75-fb8d-4cc2-bcc8-83c47d3965e4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 095473d0-4ee6-42f5-bcf0-ff52bb8660ef kind: Event lastTimestamp: "2026-05-20T14:01:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:01:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:19Z" name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn.18b14ae60a31ec39 namespace: kserve-ci-e2e-test resourceVersion: "15317" uid: 0107c6fb-41e2-4995-a949-e5396c5fa034 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: 095473d0-4ee6-42f5-bcf0-ff52bb8660ef kind: Event lastTimestamp: "2026-05-20T14:01:22Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:22Z" name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn.18b14ae69b8d6701 namespace: kserve-ci-e2e-test resourceVersion: "15373" uid: eb49031f-c938-4597-9351-7cc9de1c5b47 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f namespace: kserve-ci-e2e-test resourceVersion: "14248" uid: 2df350fc-c2e3-4503-8aeb-b42bd1165f2e kind: Event lastTimestamp: "2026-05-20T13:59:29Z" message: 'Created pod: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f-vcjhn' metadata: creationTimestamp: "2026-05-20T13:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T13:59:29Z" name: isvc-xgboost-graph-raw-4de28-predictor-76576dd44f.18b14acc5972cb75 namespace: kserve-ci-e2e-test resourceVersion: "14265" uid: 5703096b-af40-49fe-b2e8-65a3f1446d44 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T13:59:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-4de28-predictor namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 2c2cfbf2-fb20-4224-aa14-e8ed3cc05531 kind: Event lastTimestamp: "2026-05-20T13:59:29Z" message: Scaled up replica set isvc-xgboost-graph-raw-4de28-predictor-76576dd44f from 0 to 1 metadata: creationTimestamp: "2026-05-20T13:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T13:59:29Z" name: isvc-xgboost-graph-raw-4de28-predictor.18b14acc55cfb67c namespace: kserve-ci-e2e-test resourceVersion: "14251" uid: 0de5f55b-1b48-482b-8e62-31a718c7a73d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T13:59:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-4de28-predictor namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: e21b59e1-cf3f-4a2e-8b82-1302505ee8a8 kind: Event lastTimestamp: "2026-05-20T14:00:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T13:59:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:00:29Z" name: isvc-xgboost-graph-raw-4de28-predictor.18b14acfd44bc866 namespace: kserve-ci-e2e-test resourceVersion: "14785" uid: 4a450a3f-893a-4136-b9eb-301e9e7a5f7c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T13:59:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-4de28-predictor namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: e21b59e1-cf3f-4a2e-8b82-1302505ee8a8 kind: Event lastTimestamp: "2026-05-20T14:00:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T13:59:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:00:29Z" name: isvc-xgboost-graph-raw-4de28-predictor.18b14acfd4b15e70 namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 0c689af4-85e4-45d1-8039-930cebd4673d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:00:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-4de28-predictor namespace: kserve-ci-e2e-test resourceVersion: "14439" uid: e21b59e1-cf3f-4a2e-8b82-1302505ee8a8 kind: Event lastTimestamp: "2026-05-20T14:01:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:14Z" name: isvc-xgboost-graph-raw-4de28-predictor.18b14addd0196fe6 namespace: kserve-ci-e2e-test resourceVersion: "15166" uid: bd70b2bc-96fe-4134-8d3e-270ad2098a34 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:00:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-4de28-predictor namespace: kserve-ci-e2e-test resourceVersion: "14439" uid: e21b59e1-cf3f-4a2e-8b82-1302505ee8a8 kind: Event lastTimestamp: "2026-05-20T14:01:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:14Z" name: isvc-xgboost-graph-raw-4de28-predictor.18b14addd08a4b0e namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: 434dabd3-2a15-432b-847f-54752fcb2742 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-20T13:59:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-4de28 namespace: kserve-ci-e2e-test resourceVersion: "14233" uid: f92d0ff1-4cfc-4225-b36e-5879f32b6ba0 kind: Event lastTimestamp: "2026-05-20T14:01:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T13:59:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:01:02Z" name: isvc-xgboost-graph-raw-4de28.18b14acc55a580b0 namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: 47f06235-bbb9-4e3c-9362-e2efafb6a0ee reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-4de28 namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: f92d0ff1-4cfc-4225-b36e-5879f32b6ba0 kind: Event lastTimestamp: "2026-05-20T14:01:02Z" message: InferenceService [isvc-xgboost-graph-raw-4de28] is Ready metadata: creationTimestamp: "2026-05-20T14:01:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:01:02Z" name: isvc-xgboost-graph-raw-4de28.18b14ae1f8d5c26d namespace: kserve-ci-e2e-test resourceVersion: "15037" uid: 373c9a45-b57d-42f2-9792-118592703731 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:01:20.011699Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 75fcc3b2-687c-4381-816d-dc512d05f4bb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6 to ip-10-0-140-251.ec2.internal metadata: creationTimestamp: "2026-05-20T14:01:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:01:20Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6.18b14ae61f098db4 namespace: kserve-ci-e2e-test resourceVersion: "15327" uid: fa221ac7-47f1-4420-b061-1f11d9da01aa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-c8868bc95-7kctc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6 namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: 75fcc3b2-687c-4381-816d-dc512d05f4bb kind: Event lastTimestamp: "2026-05-20T14:01:20Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:01:20Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6.18b14ae6392fe8f3 namespace: kserve-ci-e2e-test resourceVersion: "15339" uid: 729e5e8a-75d7-4f02-87dd-9401c355713a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 75fcc3b2-687c-4381-816d-dc512d05f4bb kind: Event lastTimestamp: "2026-05-20T14:01:20Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0459e31fff147b1f9487117ac00a86221572ce2251d3068d36cf76ed2126ea4f" already present on machine metadata: creationTimestamp: "2026-05-20T14:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:20Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6.18b14ae63b007a18 namespace: kserve-ci-e2e-test resourceVersion: "15341" uid: 2822787c-2be3-4062-93d8-8522f9ba6281 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 75fcc3b2-687c-4381-816d-dc512d05f4bb kind: Event lastTimestamp: "2026-05-20T14:01:20Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T14:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:20Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6.18b14ae63e5a1133 namespace: kserve-ci-e2e-test resourceVersion: "15342" uid: 6d31ed29-27fa-4dd0-a808-5a5cdcd47fcb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 75fcc3b2-687c-4381-816d-dc512d05f4bb kind: Event lastTimestamp: "2026-05-20T14:01:20Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T14:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:20Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6.18b14ae63ed3e68e namespace: kserve-ci-e2e-test resourceVersion: "15343" uid: 9be26ea8-9880-4d2e-b616-3a66cca399b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 75fcc3b2-687c-4381-816d-dc512d05f4bb kind: Event lastTimestamp: "2026-05-20T14:01:24Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-05-20T14:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:24Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6.18b14ae724181009 namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 84f67928-0409-4e6e-85c2-2bd4d13a9c88 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 75fcc3b2-687c-4381-816d-dc512d05f4bb kind: Event lastTimestamp: "2026-05-20T14:01:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:24Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6.18b14ae72be5a887 namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: c20656c1-1104-4247-b278-65d6327aa900 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 75fcc3b2-687c-4381-816d-dc512d05f4bb kind: Event lastTimestamp: "2026-05-20T14:01:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:24Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6.18b14ae72c6a07d8 namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: a30ca384-72aa-4821-a973-87ab61838ae8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 75fcc3b2-687c-4381-816d-dc512d05f4bb kind: Event lastTimestamp: "2026-05-20T14:01:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:24Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6.18b14ae72c9041dd namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 7c1e0735-e09c-4f01-b7be-deb2f6f4c107 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 75fcc3b2-687c-4381-816d-dc512d05f4bb kind: Event lastTimestamp: "2026-05-20T14:01:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:24Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6.18b14ae730673965 namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 10154a0c-a0b8-441d-9a5f-c1e43a6ece63 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 75fcc3b2-687c-4381-816d-dc512d05f4bb kind: Event lastTimestamp: "2026-05-20T14:01:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:24Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6.18b14ae730e0f15d namespace: kserve-ci-e2e-test resourceVersion: "15412" uid: b156ef00-86d8-44a2-813b-2598306cd77a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T14:01:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 75fcc3b2-687c-4381-816d-dc512d05f4bb kind: Event lastTimestamp: "2026-05-20T14:03:01Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:01Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6.18b14ae760ef981e namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: bcd197db-6345-4a45-82f7-c4db4394383e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 75fcc3b2-687c-4381-816d-dc512d05f4bb kind: Event lastTimestamp: "2026-05-20T14:02:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:02:59Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6.18b14afd5e2327dc namespace: kserve-ci-e2e-test resourceVersion: "16261" uid: 92379781-96a4-4c0f-97f5-790463552943 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 75fcc3b2-687c-4381-816d-dc512d05f4bb kind: Event lastTimestamp: "2026-05-20T14:02:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:02:59Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6.18b14afd5e23bfe0 namespace: kserve-ci-e2e-test resourceVersion: "16263" uid: 7d5521c1-8ead-4773-aa3f-69dad5d1ac48 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 75fcc3b2-687c-4381-816d-dc512d05f4bb kind: Event lastTimestamp: "2026-05-20T14:03:01Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:01Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6.18b14afdbb792b37 namespace: kserve-ci-e2e-test resourceVersion: "16283" uid: 2ef608f8-5951-409c-b8ac-f8173332ebdc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f namespace: kserve-ci-e2e-test resourceVersion: "15285" uid: c871b1e1-bad7-413d-a20e-d8cbb2d33208 kind: Event lastTimestamp: "2026-05-20T14:01:20Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f-g2xs6' metadata: creationTimestamp: "2026-05-20T14:01:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:20Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f.18b14ae61e9830d1 namespace: kserve-ci-e2e-test resourceVersion: "15324" uid: ed35b3a8-d0b4-42e6-9c31-0b8e4d52ba30 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-de018-predictor namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: bf3b159c-a104-4758-9fd3-02859e44ad4f kind: Event lastTimestamp: "2026-05-20T14:01:19Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-de018-predictor-5b84fff46f from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:01:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:19Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor.18b14ae60499e75b namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 53712167-bf24-4f61-bb0a-49ab42938e90 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:01:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-de018-predictor namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: 06896b8c-3f03-4dab-bf17-aa019970425b kind: Event lastTimestamp: "2026-05-20T14:01:49Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:01:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:49Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor.18b14ae9830ea76e namespace: kserve-ci-e2e-test resourceVersion: "15623" uid: 73376312-89e8-4e8d-bb57-59ea5263e327 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:01:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-de018-predictor namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: 06896b8c-3f03-4dab-bf17-aa019970425b kind: Event lastTimestamp: "2026-05-20T14:01:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:01:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:49Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor.18b14ae983811fe0 namespace: kserve-ci-e2e-test resourceVersion: "15624" uid: 6de293de-8bae-4c76-8350-a391505c32b3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:02:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-de018-predictor namespace: kserve-ci-e2e-test resourceVersion: "15503" uid: 06896b8c-3f03-4dab-bf17-aa019970425b kind: Event lastTimestamp: "2026-05-20T14:02:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:02:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:02:49Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor.18b14af0814a81c7 namespace: kserve-ci-e2e-test resourceVersion: "16093" uid: 2d75543f-7636-4fe1-a92a-d79a49d93d4f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:02:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-de018-predictor namespace: kserve-ci-e2e-test resourceVersion: "15503" uid: 06896b8c-3f03-4dab-bf17-aa019970425b kind: Event lastTimestamp: "2026-05-20T14:02:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:02:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:02:49Z" name: isvc-xgboost-graph-raw-hpa-de018-predictor.18b14af081d30f6c namespace: kserve-ci-e2e-test resourceVersion: "16095" uid: 8418c755-419e-4bfb-ad58-ee8d9c393131 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T14:01:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-de018 namespace: kserve-ci-e2e-test resourceVersion: "15268" uid: 08ea1a1b-7212-4baa-a65c-24b175465288 kind: Event lastTimestamp: "2026-05-20T14:02:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:01:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:02:31Z" name: isvc-xgboost-graph-raw-hpa-de018.18b14ae604660f61 namespace: kserve-ci-e2e-test resourceVersion: "15929" uid: c11fabf1-21af-4716-8f04-74f1b2712f5c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-de018 namespace: kserve-ci-e2e-test resourceVersion: "15268" uid: 08ea1a1b-7212-4baa-a65c-24b175465288 kind: Event lastTimestamp: "2026-05-20T14:01:19Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-de018": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-de018": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:01:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:01:19Z" name: isvc-xgboost-graph-raw-hpa-de018.18b14ae604e797da namespace: kserve-ci-e2e-test resourceVersion: "15292" uid: e1695bb9-a433-40d2-b561-cc4cae7a8c89 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-de018 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: 08ea1a1b-7212-4baa-a65c-24b175465288 kind: Event lastTimestamp: "2026-05-20T14:02:31Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-de018] is Ready metadata: creationTimestamp: "2026-05-20T14:02:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:02:31Z" name: isvc-xgboost-graph-raw-hpa-de018.18b14af6c4e54d6a namespace: kserve-ci-e2e-test resourceVersion: "15928" uid: 5cd52300-7bb4-4d42-a84f-827d70910b75 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:02:59.810582Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 9cae34c1-3cb9-452b-9ec4-3cbd4077479a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d to ip-10-0-140-251.ec2.internal metadata: creationTimestamp: "2026-05-20T14:02:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:02:59Z" name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d.18b14afd5b83abe0 namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: e04900de-527e-4fd7-8a9e-785b4bdaebb7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-c8868bc95-7kctc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:00Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 9cae34c1-3cb9-452b-9ec4-3cbd4077479a kind: Event lastTimestamp: "2026-05-20T14:03:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-raw-cbb25-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:03:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:00Z" name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d.18b14afd6746b345 namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: c849bd45-96db-4fee-a28b-1bdfd3800d27 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:00Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: 9cae34c1-3cb9-452b-9ec4-3cbd4077479a kind: Event lastTimestamp: "2026-05-20T14:03:00Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:03:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:03:00Z" name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d.18b14afd9939389f namespace: kserve-ci-e2e-test resourceVersion: "16274" uid: bc762caa-fb56-40f0-a086-bf2f8025ce3a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 9cae34c1-3cb9-452b-9ec4-3cbd4077479a kind: Event lastTimestamp: "2026-05-20T14:03:00Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-05-20T14:03:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:00Z" name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d.18b14afd9a8da1e5 namespace: kserve-ci-e2e-test resourceVersion: "16276" uid: 1e0b5fdb-0da7-44ce-8452-1aa5973892ca reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 9cae34c1-3cb9-452b-9ec4-3cbd4077479a kind: Event lastTimestamp: "2026-05-20T14:03:01Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 930ms (930ms including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-05-20T14:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:01Z" name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d.18b14afdd20acf3f namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: e44fc0a3-7348-4cd0-8014-688cdd9a64f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 9cae34c1-3cb9-452b-9ec4-3cbd4077479a kind: Event lastTimestamp: "2026-05-20T14:03:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:01Z" name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d.18b14afdd5c78c41 namespace: kserve-ci-e2e-test resourceVersion: "16292" uid: d27251da-0a44-4d7c-99de-72af5acd3758 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 9cae34c1-3cb9-452b-9ec4-3cbd4077479a kind: Event lastTimestamp: "2026-05-20T14:03:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:01Z" name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d.18b14afdd6453315 namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: 7289e9c7-53c3-4518-a483-822ba98e4982 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 9cae34c1-3cb9-452b-9ec4-3cbd4077479a kind: Event lastTimestamp: "2026-05-20T14:03:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:01Z" name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d.18b14afdd6531659 namespace: kserve-ci-e2e-test resourceVersion: "16294" uid: 7135c557-0916-4eb9-951c-c28c0a14462a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 9cae34c1-3cb9-452b-9ec4-3cbd4077479a kind: Event lastTimestamp: "2026-05-20T14:03:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:01Z" name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d.18b14afdd9ba0660 namespace: kserve-ci-e2e-test resourceVersion: "16295" uid: 9118bfe7-11ae-4533-bb29-fc233772b90c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 9cae34c1-3cb9-452b-9ec4-3cbd4077479a kind: Event lastTimestamp: "2026-05-20T14:03:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:01Z" name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d.18b14afdda369416 namespace: kserve-ci-e2e-test resourceVersion: "16296" uid: 35491eb4-3f95-425c-bb5d-621130e27848 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 9cae34c1-3cb9-452b-9ec4-3cbd4077479a kind: Event lastTimestamp: "2026-05-20T14:04:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:45Z" name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d.18b14b15e16c4f9d namespace: kserve-ci-e2e-test resourceVersion: "17209" uid: 99cd1365-cf34-408d-a21b-8a7fed9ed2e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:04:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 9cae34c1-3cb9-452b-9ec4-3cbd4077479a kind: Event lastTimestamp: "2026-05-20T14:04:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:04:45Z" name: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d.18b14b15e16ccbde namespace: kserve-ci-e2e-test resourceVersion: "17210" uid: fd67ca87-1b93-42a2-97b3-0d2eae67cb59 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-cbb25-predictor-74678f7ffd namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: 0a64c7ef-1957-44a9-9e63-3773c006bd43 kind: Event lastTimestamp: "2026-05-20T14:02:59Z" message: 'Created pod: message-dumper-raw-cbb25-predictor-74678f7ffd-spr2d' metadata: creationTimestamp: "2026-05-20T14:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:02:59Z" name: message-dumper-raw-cbb25-predictor-74678f7ffd.18b14afd5aef5bf2 namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: e0b5c6c9-33a9-4df2-a2e4-3fa3aae7892f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-cbb25-predictor namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: 35b93f09-34c4-43dd-b900-8c69db09ddbd kind: Event lastTimestamp: "2026-05-20T14:02:59Z" message: Scaled up replica set message-dumper-raw-cbb25-predictor-74678f7ffd from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:02:59Z" name: message-dumper-raw-cbb25-predictor.18b14afd57cefb07 namespace: kserve-ci-e2e-test resourceVersion: "16222" uid: dcc50669-b38a-441f-aeb6-e54cd3f3905f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:03:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-cbb25-predictor namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: c0912e53-a0a6-4c1a-89d4-7081d50ce94d kind: Event lastTimestamp: "2026-05-20T14:03:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:44Z" name: message-dumper-raw-cbb25-predictor.18b14b00d64e8bf8 namespace: kserve-ci-e2e-test resourceVersion: "16698" uid: cdf46db3-063f-486e-bb02-d659d547e295 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:03:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-cbb25-predictor namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: c0912e53-a0a6-4c1a-89d4-7081d50ce94d kind: Event lastTimestamp: "2026-05-20T14:03:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:03:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:03:44Z" name: message-dumper-raw-cbb25-predictor.18b14b00d6b6e683 namespace: kserve-ci-e2e-test resourceVersion: "16699" uid: a2ebd54d-198d-4222-8974-c1f39cd98271 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-20T14:02:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-cbb25 namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: 9af74984-be55-43e0-bc12-b0b381c6bb9d kind: Event lastTimestamp: "2026-05-20T14:03:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:03:09Z" name: message-dumper-raw-cbb25.18b14afd57a300ff namespace: kserve-ci-e2e-test resourceVersion: "16371" uid: 168ba4a4-4c2d-4310-beec-1d154e2e79f3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-cbb25 namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: 9af74984-be55-43e0-bc12-b0b381c6bb9d kind: Event lastTimestamp: "2026-05-20T14:02:59Z" message: 'Failed to update status for InferenceService "message-dumper-raw-cbb25": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-cbb25": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:02:59Z" name: message-dumper-raw-cbb25.18b14afd58741d67 namespace: kserve-ci-e2e-test resourceVersion: "16225" uid: ac584975-5dc7-4d6a-8e7e-ca09b3c5707f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:03:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-cbb25 namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: 9af74984-be55-43e0-bc12-b0b381c6bb9d kind: Event lastTimestamp: "2026-05-20T14:03:09Z" message: InferenceService [message-dumper-raw-cbb25] is Ready metadata: creationTimestamp: "2026-05-20T14:03:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:03:09Z" name: message-dumper-raw-cbb25.18b14affb149cfe4 namespace: kserve-ci-e2e-test resourceVersion: "16370" uid: 5b23d37d-2660-4df9-af43-23af43910031 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:01:09.294517Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-4de28-7486fff99b-44grl namespace: kserve-ci-e2e-test resourceVersion: "15103" uid: 5e3d8b78-1141-4a98-b310-671927dcfc91 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-4de28-7486fff99b-44grl to ip-10-0-130-134.ec2.internal metadata: creationTimestamp: "2026-05-20T14:01:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:01:09Z" name: model-chainer-raw-4de28-7486fff99b-44grl.18b14ae3a03e55ce namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: a4b6fd0f-b67b-4ca8-8a04-fd5fc9af3856 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-c8868bc95-7kctc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:09Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-4de28-7486fff99b-44grl namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: 5e3d8b78-1141-4a98-b310-671927dcfc91 kind: Event lastTimestamp: "2026-05-20T14:01:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-4de28-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:09Z" name: model-chainer-raw-4de28-7486fff99b-44grl.18b14ae3aa9505f3 namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: 9feb29d7-3516-47e2-83ba-fd26005605b3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-134.ec2.internal source: component: kubelet host: ip-10-0-130-134.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:10Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-4de28-7486fff99b-44grl namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 5e3d8b78-1141-4a98-b310-671927dcfc91 kind: Event lastTimestamp: "2026-05-20T14:01:10Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:01:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:01:10Z" name: model-chainer-raw-4de28-7486fff99b-44grl.18b14ae3dd484b4d namespace: kserve-ci-e2e-test resourceVersion: "15128" uid: c9a39f76-c24a-403c-a17b-7dd60dd7116b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-4de28} kind: Pod name: model-chainer-raw-4de28-7486fff99b-44grl namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: 5e3d8b78-1141-4a98-b310-671927dcfc91 kind: Event lastTimestamp: "2026-05-20T14:01:10Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:04926a9d7185ff179745ae32ed77ae962904508dbcb7541e9bf600392af6f503" metadata: creationTimestamp: "2026-05-20T14:01:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:10Z" name: model-chainer-raw-4de28-7486fff99b-44grl.18b14ae3deb0a75a namespace: kserve-ci-e2e-test resourceVersion: "15130" uid: c1811522-5830-4f5e-a195-f2c75c67c85b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-134.ec2.internal source: component: kubelet host: ip-10-0-130-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-4de28} kind: Pod name: model-chainer-raw-4de28-7486fff99b-44grl namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: 5e3d8b78-1141-4a98-b310-671927dcfc91 kind: Event lastTimestamp: "2026-05-20T14:01:13Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:04926a9d7185ff179745ae32ed77ae962904508dbcb7541e9bf600392af6f503" in 2.701s (2.701s including waiting). Image size: 218002168 bytes.' metadata: creationTimestamp: "2026-05-20T14:01:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:13Z" name: model-chainer-raw-4de28-7486fff99b-44grl.18b14ae47fba3e7e namespace: kserve-ci-e2e-test resourceVersion: "15149" uid: 63cb2cad-c027-484c-82a8-feda34db3d6c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-134.ec2.internal source: component: kubelet host: ip-10-0-130-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-4de28} kind: Pod name: model-chainer-raw-4de28-7486fff99b-44grl namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: 5e3d8b78-1141-4a98-b310-671927dcfc91 kind: Event lastTimestamp: "2026-05-20T14:01:13Z" message: 'Created container: model-chainer-raw-4de28' metadata: creationTimestamp: "2026-05-20T14:01:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:13Z" name: model-chainer-raw-4de28-7486fff99b-44grl.18b14ae48310ade5 namespace: kserve-ci-e2e-test resourceVersion: "15150" uid: 2bb38382-bf31-4aa8-9c1d-68a2bd8e7fd4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-134.ec2.internal source: component: kubelet host: ip-10-0-130-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-4de28} kind: Pod name: model-chainer-raw-4de28-7486fff99b-44grl namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: 5e3d8b78-1141-4a98-b310-671927dcfc91 kind: Event lastTimestamp: "2026-05-20T14:01:13Z" message: Started container model-chainer-raw-4de28 metadata: creationTimestamp: "2026-05-20T14:01:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:13Z" name: model-chainer-raw-4de28-7486fff99b-44grl.18b14ae483825d1f namespace: kserve-ci-e2e-test resourceVersion: "15151" uid: 3a27ab85-14dc-48b3-af87-ca2b831d3390 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-134.ec2.internal source: component: kubelet host: ip-10-0-130-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-4de28} kind: Pod name: model-chainer-raw-4de28-7486fff99b-44grl namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: 5e3d8b78-1141-4a98-b310-671927dcfc91 kind: Event lastTimestamp: "2026-05-20T14:01:19Z" message: Stopping container model-chainer-raw-4de28 metadata: creationTimestamp: "2026-05-20T14:01:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:19Z" name: model-chainer-raw-4de28-7486fff99b-44grl.18b14ae5f78fb044 namespace: kserve-ci-e2e-test resourceVersion: "15226" uid: 4b4d525d-6773-4e32-bc89-e8d029536723 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-134.ec2.internal source: component: kubelet host: ip-10-0-130-134.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:01:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-4de28} kind: Pod name: model-chainer-raw-4de28-7486fff99b-44grl namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: 5e3d8b78-1141-4a98-b310-671927dcfc91 kind: Event lastTimestamp: "2026-05-20T14:01:49Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T14:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:01:49Z" name: model-chainer-raw-4de28-7486fff99b-44grl.18b14ae7189227f6 namespace: kserve-ci-e2e-test resourceVersion: "15612" uid: 5052163b-898a-48c3-b5ea-53515a89109e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-134.ec2.internal source: component: kubelet host: ip-10-0-130-134.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-4de28-7486fff99b namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: 881f571d-60cb-4cef-bbe2-e5b7ba9ef59e kind: Event lastTimestamp: "2026-05-20T14:01:09Z" message: 'Created pod: model-chainer-raw-4de28-7486fff99b-44grl' metadata: creationTimestamp: "2026-05-20T14:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:09Z" name: model-chainer-raw-4de28-7486fff99b.18b14ae39fad3eca namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 41fe20d3-39f6-46b3-8a28-8d436adafb33 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-4de28 namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: 0609c1a4-4c46-4836-8d28-a8ff475a5272 kind: Event lastTimestamp: "2026-05-20T14:01:09Z" message: Scaled up replica set model-chainer-raw-4de28-7486fff99b from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:01:09Z" name: model-chainer-raw-4de28.18b14ae39ebf676d namespace: kserve-ci-e2e-test resourceVersion: "15101" uid: c5d13aef-8698-4ad9-abb2-7f0fbb29cacc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:01:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-4de28 namespace: kserve-ci-e2e-test resourceVersion: "15114" uid: 0d39651a-0003-49c5-9561-0948a654ca10 kind: Event lastTimestamp: "2026-05-20T14:01:09Z" message: InferenceGraph [model-chainer-raw-4de28] is Ready metadata: creationTimestamp: "2026-05-20T14:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:01:09Z" name: model-chainer-raw-4de28.18b14ae3a38b1d03 namespace: kserve-ci-e2e-test resourceVersion: "15116" uid: 3a3de8a3-82cf-4697-84b2-5013ead978eb reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:02:49.571808Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-de018-75bdf5d596-zmplb namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: 317eb2e7-af0f-4681-8d97-7d6ce52d5ee6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-de018-75bdf5d596-zmplb to ip-10-0-130-134.ec2.internal metadata: creationTimestamp: "2026-05-20T14:02:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:02:49Z" name: model-chainer-raw-hpa-de018-75bdf5d596-zmplb.18b14afaf93c65b6 namespace: kserve-ci-e2e-test resourceVersion: "16083" uid: 9b5d4f6a-6a91-4e3e-a7d2-9bd0558907f7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-c8868bc95-7kctc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:49Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-de018-75bdf5d596-zmplb namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: 317eb2e7-af0f-4681-8d97-7d6ce52d5ee6 kind: Event lastTimestamp: "2026-05-20T14:02:49Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:02:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:02:49Z" name: model-chainer-raw-hpa-de018-75bdf5d596-zmplb.18b14afb127a7565 namespace: kserve-ci-e2e-test resourceVersion: "16101" uid: a0fd7a91-d04d-41f5-9b5f-3a3325c3533a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-de018} kind: Pod name: model-chainer-raw-hpa-de018-75bdf5d596-zmplb namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: 317eb2e7-af0f-4681-8d97-7d6ce52d5ee6 kind: Event lastTimestamp: "2026-05-20T14:02:50Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:04926a9d7185ff179745ae32ed77ae962904508dbcb7541e9bf600392af6f503" already present on machine metadata: creationTimestamp: "2026-05-20T14:02:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:02:50Z" name: model-chainer-raw-hpa-de018-75bdf5d596-zmplb.18b14afb13b6f3d9 namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: 9ba25dba-976f-454c-83b5-94009e7cd331 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-134.ec2.internal source: component: kubelet host: ip-10-0-130-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-de018} kind: Pod name: model-chainer-raw-hpa-de018-75bdf5d596-zmplb namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: 317eb2e7-af0f-4681-8d97-7d6ce52d5ee6 kind: Event lastTimestamp: "2026-05-20T14:02:50Z" message: 'Created container: model-chainer-raw-hpa-de018' metadata: creationTimestamp: "2026-05-20T14:02:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:02:50Z" name: model-chainer-raw-hpa-de018-75bdf5d596-zmplb.18b14afb169ecf2d namespace: kserve-ci-e2e-test resourceVersion: "16104" uid: e8a2a498-08dc-414a-be32-7c1ff2e35961 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-134.ec2.internal source: component: kubelet host: ip-10-0-130-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-de018} kind: Pod name: model-chainer-raw-hpa-de018-75bdf5d596-zmplb namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: 317eb2e7-af0f-4681-8d97-7d6ce52d5ee6 kind: Event lastTimestamp: "2026-05-20T14:02:50Z" message: Started container model-chainer-raw-hpa-de018 metadata: creationTimestamp: "2026-05-20T14:02:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:02:50Z" name: model-chainer-raw-hpa-de018-75bdf5d596-zmplb.18b14afb1711e90b namespace: kserve-ci-e2e-test resourceVersion: "16105" uid: 6660a3bc-1efd-4006-a722-feb7b9c99b45 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-134.ec2.internal source: component: kubelet host: ip-10-0-130-134.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-de018} kind: Pod name: model-chainer-raw-hpa-de018-75bdf5d596-zmplb namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: 317eb2e7-af0f-4681-8d97-7d6ce52d5ee6 kind: Event lastTimestamp: "2026-05-20T14:02:59Z" message: Stopping container model-chainer-raw-hpa-de018 metadata: creationTimestamp: "2026-05-20T14:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:02:59Z" name: model-chainer-raw-hpa-de018-75bdf5d596-zmplb.18b14afd5174d75c namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 21057377-ec96-4a1f-8376-cdb570f63dbe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-134.ec2.internal source: component: kubelet host: ip-10-0-130-134.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-20T14:03:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-de018} kind: Pod name: model-chainer-raw-hpa-de018-75bdf5d596-zmplb namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: 317eb2e7-af0f-4681-8d97-7d6ce52d5ee6 kind: Event lastTimestamp: "2026-05-20T14:03:26Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-20T14:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:03:26Z" name: model-chainer-raw-hpa-de018-75bdf5d596-zmplb.18b14afdc044807f namespace: kserve-ci-e2e-test resourceVersion: "16557" uid: 0304f608-e646-4f60-bab2-ae1656c1a7e9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-134.ec2.internal source: component: kubelet host: ip-10-0-130-134.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-de018-75bdf5d596 namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: 5db70e9d-f5b3-47b5-ac62-a304d26e341b kind: Event lastTimestamp: "2026-05-20T14:02:49Z" message: 'Created pod: model-chainer-raw-hpa-de018-75bdf5d596-zmplb' metadata: creationTimestamp: "2026-05-20T14:02:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:02:49Z" name: model-chainer-raw-hpa-de018-75bdf5d596.18b14afaf8ce3953 namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: 9e87c204-e9d1-453c-a68f-530fcaed8930 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-de018 namespace: kserve-ci-e2e-test resourceVersion: "16071" uid: 2aad68fd-c15f-4da9-b3ff-3bdb8e27279d kind: Event lastTimestamp: "2026-05-20T14:02:49Z" message: Scaled up replica set model-chainer-raw-hpa-de018-75bdf5d596 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:02:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:02:49Z" name: model-chainer-raw-hpa-de018.18b14afaf80b94a6 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: aa779665-8df7-42fa-b197-3229b28fbdd2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:02:49Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-de018 namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: 285eeb0f-0776-4692-8ec2-3b635d2fd8fb kind: Event lastTimestamp: "2026-05-20T14:02:49Z" message: InferenceGraph [model-chainer-raw-hpa-de018] is Ready metadata: creationTimestamp: "2026-05-20T14:02:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:02:49Z" name: model-chainer-raw-hpa-de018.18b14afafd950397 namespace: kserve-ci-e2e-test resourceVersion: "16092" uid: 3073faa2-da5b-4064-8108-f078faf3ae8d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:08:59.776246Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74 namespace: kserve-ci-e2e-test resourceVersion: "19359" uid: b4bf8885-c618-4380-bbde-a36691630392 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-1b556-predictor-6bfd95dfc9-sns74 to ip-10-0-140-251.ec2.internal metadata: creationTimestamp: "2026-05-20T14:08:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:08:59Z" name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74.18b14b512b23d0f2 namespace: kserve-ci-e2e-test resourceVersion: "19363" uid: dddf9205-976a-4480-a7ff-8074b5523d9c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-c8868bc95-7kctc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:08:59Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74 namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: b4bf8885-c618-4380-bbde-a36691630392 kind: Event lastTimestamp: "2026-05-20T14:08:59Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-1b556-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-20T14:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:08:59Z" name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74.18b14b51357c43be namespace: kserve-ci-e2e-test resourceVersion: "19375" uid: 18be7c42-68b1-472e-ac13-9d80779b4b4d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:09:00Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74 namespace: kserve-ci-e2e-test resourceVersion: "19369" uid: b4bf8885-c618-4380-bbde-a36691630392 kind: Event lastTimestamp: "2026-05-20T14:09:00Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:09:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:09:00Z" name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74.18b14b51688d5a29 namespace: kserve-ci-e2e-test resourceVersion: "19387" uid: a7606000-6d12-48e3-a7fc-526d8d39ccd3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:09:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74 namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: b4bf8885-c618-4380-bbde-a36691630392 kind: Event lastTimestamp: "2026-05-20T14:09:00Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0459e31fff147b1f9487117ac00a86221572ce2251d3068d36cf76ed2126ea4f" already present on machine metadata: creationTimestamp: "2026-05-20T14:09:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:09:00Z" name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74.18b14b5169ddf164 namespace: kserve-ci-e2e-test resourceVersion: "19389" uid: 097afd7a-fbaf-4eb8-abee-9573efe28593 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:09:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74 namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: b4bf8885-c618-4380-bbde-a36691630392 kind: Event lastTimestamp: "2026-05-20T14:09:00Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T14:09:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:09:00Z" name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74.18b14b516d5293ec namespace: kserve-ci-e2e-test resourceVersion: "19391" uid: 89999f74-a6ea-4367-ae3c-d0277d643005 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:09:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74 namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: b4bf8885-c618-4380-bbde-a36691630392 kind: Event lastTimestamp: "2026-05-20T14:09:00Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T14:09:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:09:00Z" name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74.18b14b516dcd1e58 namespace: kserve-ci-e2e-test resourceVersion: "19392" uid: 15041795-6d37-4988-9252-c3d031612d11 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:09:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74 namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: b4bf8885-c618-4380-bbde-a36691630392 kind: Event lastTimestamp: "2026-05-20T14:09:04Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:09:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:09:04Z" name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74.18b14b5261eda21a namespace: kserve-ci-e2e-test resourceVersion: "19431" uid: aae5418d-8298-49fb-b3cf-8fad9e8dc8fb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:09:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74 namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: b4bf8885-c618-4380-bbde-a36691630392 kind: Event lastTimestamp: "2026-05-20T14:09:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:09:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:09:05Z" name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74.18b14b526541ab81 namespace: kserve-ci-e2e-test resourceVersion: "19437" uid: fe9312b4-d304-4fdb-8551-a751255d9718 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:09:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74 namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: b4bf8885-c618-4380-bbde-a36691630392 kind: Event lastTimestamp: "2026-05-20T14:09:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:09:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:09:05Z" name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74.18b14b5265bc9bec namespace: kserve-ci-e2e-test resourceVersion: "19438" uid: 9d306746-597d-4528-9be7-793a5885091c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:09:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74 namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: b4bf8885-c618-4380-bbde-a36691630392 kind: Event lastTimestamp: "2026-05-20T14:09:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:09:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:09:05Z" name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74.18b14b5265c956b1 namespace: kserve-ci-e2e-test resourceVersion: "19439" uid: c9d75737-aa73-4f80-a854-9771becd9bf3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:09:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74 namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: b4bf8885-c618-4380-bbde-a36691630392 kind: Event lastTimestamp: "2026-05-20T14:09:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:09:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:09:05Z" name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74.18b14b526a1270f2 namespace: kserve-ci-e2e-test resourceVersion: "19441" uid: 9e25e70e-0439-486a-8cd4-fff037128266 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:09:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74 namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: b4bf8885-c618-4380-bbde-a36691630392 kind: Event lastTimestamp: "2026-05-20T14:09:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:09:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:09:05Z" name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74.18b14b526a9f0eef namespace: kserve-ci-e2e-test resourceVersion: "19442" uid: d54755cb-114f-40d5-9f2c-fa91cc60728e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T14:09:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74 namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: b4bf8885-c618-4380-bbde-a36691630392 kind: Event lastTimestamp: "2026-05-20T14:10:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:09:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:10:22Z" name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74.18b14b529ddf6ba7 namespace: kserve-ci-e2e-test resourceVersion: "20068" uid: f074142e-6f82-45b4-a448-5079f0646e60 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:10:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74 namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: b4bf8885-c618-4380-bbde-a36691630392 kind: Event lastTimestamp: "2026-05-20T14:10:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:10:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:10:19Z" name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74.18b14b63d23b82fa namespace: kserve-ci-e2e-test resourceVersion: "20011" uid: f17d5c69-d014-4f2e-8262-550775b21934 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:10:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74 namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: b4bf8885-c618-4380-bbde-a36691630392 kind: Event lastTimestamp: "2026-05-20T14:10:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:10:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:10:19Z" name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74.18b14b63d23bbdd9 namespace: kserve-ci-e2e-test resourceVersion: "20013" uid: 38f5f81b-c7e6-4b9b-b5fa-cba4e72f1d1d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:10:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74 namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: b4bf8885-c618-4380-bbde-a36691630392 kind: Event lastTimestamp: "2026-05-20T14:10:21Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:10:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:10:22Z" name: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74.18b14b644ffb7972 namespace: kserve-ci-e2e-test resourceVersion: "20067" uid: d743c21b-31cf-41d2-9e45-531de9278d6f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:08:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-1b556-predictor-6bfd95dfc9 namespace: kserve-ci-e2e-test resourceVersion: "19348" uid: db73a29e-7a14-4f81-b0de-c8097587dd09 kind: Event lastTimestamp: "2026-05-20T14:08:59Z" message: 'Created pod: raw-sklearn-1b556-predictor-6bfd95dfc9-sns74' metadata: creationTimestamp: "2026-05-20T14:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:08:59Z" name: raw-sklearn-1b556-predictor-6bfd95dfc9.18b14b512ab4af0d namespace: kserve-ci-e2e-test resourceVersion: "19360" uid: 43328256-6db5-43b7-98d4-4737b56dec00 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:08:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-1b556-predictor namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 28bee597-8ef3-44da-a46f-17a0e244a657 kind: Event lastTimestamp: "2026-05-20T14:08:59Z" message: Scaled up replica set raw-sklearn-1b556-predictor-6bfd95dfc9 from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:08:59Z" name: raw-sklearn-1b556-predictor.18b14b5127949fda namespace: kserve-ci-e2e-test resourceVersion: "19350" uid: cae2cf5f-fa62-40a6-af01-9c437e549c0f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:09:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-1b556-predictor namespace: kserve-ci-e2e-test resourceVersion: "19347" uid: f6bde329-ad92-4323-be7f-99b4896b31bb kind: Event lastTimestamp: "2026-05-20T14:09:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:09:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:09:44Z" name: raw-sklearn-1b556-predictor.18b14b54a633ca0d namespace: kserve-ci-e2e-test resourceVersion: "19723" uid: 77c4be26-299a-4ba8-90e0-22979e152209 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:09:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-1b556-predictor namespace: kserve-ci-e2e-test resourceVersion: "19347" uid: f6bde329-ad92-4323-be7f-99b4896b31bb kind: Event lastTimestamp: "2026-05-20T14:09:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:09:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:09:44Z" name: raw-sklearn-1b556-predictor.18b14b54a6987d22 namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 0860d948-4ac2-404b-a711-f59f61132a2e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:09:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-1b556-predictor namespace: kserve-ci-e2e-test resourceVersion: "19510" uid: f6bde329-ad92-4323-be7f-99b4896b31bb kind: Event lastTimestamp: "2026-05-20T14:10:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:10:14Z" name: raw-sklearn-1b556-predictor.18b14b582597972d namespace: kserve-ci-e2e-test resourceVersion: "19940" uid: 75074d4a-4548-4da1-9b78-24d04b8e1f5e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-20T14:09:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-1b556-predictor namespace: kserve-ci-e2e-test resourceVersion: "19510" uid: f6bde329-ad92-4323-be7f-99b4896b31bb kind: Event lastTimestamp: "2026-05-20T14:10:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:10:14Z" name: raw-sklearn-1b556-predictor.18b14b5825fd5ae2 namespace: kserve-ci-e2e-test resourceVersion: "19941" uid: 4adc4412-5e0b-4db4-be1b-b6c38c2345b7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-05-20T14:08:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-1b556 namespace: kserve-ci-e2e-test resourceVersion: "19336" uid: e21f88ac-8e0a-4d96-bb78-46509bd54629 kind: Event lastTimestamp: "2026-05-20T14:10:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:10:12Z" name: raw-sklearn-1b556.18b14b51276b12cf namespace: kserve-ci-e2e-test resourceVersion: "19924" uid: fc48f469-e2fe-4150-8b8b-42acd03c20aa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:08:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-1b556 namespace: kserve-ci-e2e-test resourceVersion: "19336" uid: e21f88ac-8e0a-4d96-bb78-46509bd54629 kind: Event lastTimestamp: "2026-05-20T14:08:59Z" message: 'Failed to update status for InferenceService "raw-sklearn-1b556": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-1b556": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:08:59Z" name: raw-sklearn-1b556.18b14b5127d53463 namespace: kserve-ci-e2e-test resourceVersion: "19373" uid: 55f3afa7-bce3-4de9-892c-e2fc28b44b78 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:08:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-1b556 namespace: kserve-ci-e2e-test resourceVersion: "19356" uid: e21f88ac-8e0a-4d96-bb78-46509bd54629 kind: Event lastTimestamp: "2026-05-20T14:08:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-1b556": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:08:59Z" name: raw-sklearn-1b556.18b14b512e9f2795 namespace: kserve-ci-e2e-test resourceVersion: "19374" uid: ea34c1b9-3f2a-42ed-92cc-e6fce30d26d8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:10:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-1b556 namespace: kserve-ci-e2e-test resourceVersion: "19922" uid: e21f88ac-8e0a-4d96-bb78-46509bd54629 kind: Event lastTimestamp: "2026-05-20T14:10:12Z" message: InferenceService [raw-sklearn-1b556] is Ready metadata: creationTimestamp: "2026-05-20T14:10:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:10:12Z" name: raw-sklearn-1b556.18b14b62019bcab1 namespace: kserve-ci-e2e-test resourceVersion: "19923" uid: 92c59464-495a-4ed7-9011-031c139b0ef0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-20T14:10:19.979270Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h namespace: kserve-ci-e2e-test resourceVersion: "20029" uid: 4d72a935-c239-42e7-8a84-0d60da5188db kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h to ip-10-0-140-251.ec2.internal metadata: creationTimestamp: "2026-05-20T14:10:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-20T14:10:19Z" name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h.18b14b63d79ce67f namespace: kserve-ci-e2e-test resourceVersion: "20033" uid: 5dbabc9a-02ce-4dd0-aca4-7e0f5182f8be reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-c8868bc95-7kctc source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:10:20Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: 4d72a935-c239-42e7-8a84-0d60da5188db kind: Event lastTimestamp: "2026-05-20T14:10:20Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-20T14:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-20T14:10:20Z" name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h.18b14b63f17a4a88 namespace: kserve-ci-e2e-test resourceVersion: "20043" uid: 2a111245-0112-4bcd-beb2-65298ff3a4c0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:10:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 4d72a935-c239-42e7-8a84-0d60da5188db kind: Event lastTimestamp: "2026-05-20T14:10:20Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0459e31fff147b1f9487117ac00a86221572ce2251d3068d36cf76ed2126ea4f" already present on machine metadata: creationTimestamp: "2026-05-20T14:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:10:20Z" name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h.18b14b63f2fe6c36 namespace: kserve-ci-e2e-test resourceVersion: "20046" uid: ac113a12-23b8-4824-94e1-20148fd3a045 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:10:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 4d72a935-c239-42e7-8a84-0d60da5188db kind: Event lastTimestamp: "2026-05-20T14:10:20Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-20T14:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:10:20Z" name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h.18b14b63f5d641cc namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 2126ff3d-dec0-44e3-b7fa-34afccf88304 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:10:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 4d72a935-c239-42e7-8a84-0d60da5188db kind: Event lastTimestamp: "2026-05-20T14:10:20Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-20T14:10:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:10:20Z" name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h.18b14b63f64e4720 namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: 3f11e644-78d5-4288-b7bf-7ffa2336a21c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:10:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 4d72a935-c239-42e7-8a84-0d60da5188db kind: Event lastTimestamp: "2026-05-20T14:10:24Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1501" already present on machine metadata: creationTimestamp: "2026-05-20T14:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:10:24Z" name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h.18b14b64d62fc7b3 namespace: kserve-ci-e2e-test resourceVersion: "20083" uid: 386c65fc-4822-43a1-bbbf-a08fcced6e2b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:10:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 4d72a935-c239-42e7-8a84-0d60da5188db kind: Event lastTimestamp: "2026-05-20T14:10:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-20T14:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:10:24Z" name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h.18b14b64d9b21a3a namespace: kserve-ci-e2e-test resourceVersion: "20087" uid: 8acadb67-7d41-4982-b680-b1f1e01d4bab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:10:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 4d72a935-c239-42e7-8a84-0d60da5188db kind: Event lastTimestamp: "2026-05-20T14:10:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-20T14:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:10:24Z" name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h.18b14b64da2c1531 namespace: kserve-ci-e2e-test resourceVersion: "20088" uid: fe0a16c3-a959-44ef-8d2c-bb50157023f1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:10:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 4d72a935-c239-42e7-8a84-0d60da5188db kind: Event lastTimestamp: "2026-05-20T14:10:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-20T14:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:10:24Z" name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h.18b14b64da38ce0b namespace: kserve-ci-e2e-test resourceVersion: "20089" uid: 91743f30-25d2-4e2e-94df-53ae5b73839d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:10:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 4d72a935-c239-42e7-8a84-0d60da5188db kind: Event lastTimestamp: "2026-05-20T14:10:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-20T14:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:10:24Z" name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h.18b14b64dd4ad319 namespace: kserve-ci-e2e-test resourceVersion: "20091" uid: 690c1d5f-cf50-4fd6-a56f-7f2d7ecf7ab5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:10:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 4d72a935-c239-42e7-8a84-0d60da5188db kind: Event lastTimestamp: "2026-05-20T14:10:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:10:24Z" name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h.18b14b64ddd3545b namespace: kserve-ci-e2e-test resourceVersion: "20092" uid: f0e1b731-a204-4471-a9bd-ec0ebe8147aa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-20T14:10:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 4d72a935-c239-42e7-8a84-0d60da5188db kind: Event lastTimestamp: "2026-05-20T14:11:41Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:10:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:11:41Z" name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h.18b14b651242c215 namespace: kserve-ci-e2e-test resourceVersion: "20670" uid: 604bb088-64b0-477b-b49f-e1980cd3d301 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:11:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 4d72a935-c239-42e7-8a84-0d60da5188db kind: Event lastTimestamp: "2026-05-20T14:11:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-20T14:11:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:11:40Z" name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h.18b14b767bc38daa namespace: kserve-ci-e2e-test resourceVersion: "20657" uid: 252c03d4-4242-4066-bb2b-4d1a87fe2f01 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:11:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 4d72a935-c239-42e7-8a84-0d60da5188db kind: Event lastTimestamp: "2026-05-20T14:11:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-20T14:11:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:11:40Z" name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h.18b14b767bc428c2 namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: 1ddbb790-a835-4a71-a59a-5b9e077c7845 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-20T14:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: 4d72a935-c239-42e7-8a84-0d60da5188db kind: Event lastTimestamp: "2026-05-20T14:11:46Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-20T14:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-20T14:11:46Z" name: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h.18b14b76c47efe91 namespace: kserve-ci-e2e-test resourceVersion: "20739" uid: a5a0cc85-9d45-408c-8b89-60f13aeff9a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-251.ec2.internal source: component: kubelet host: ip-10-0-140-251.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:10:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-83724-predictor-d9995568c namespace: kserve-ci-e2e-test resourceVersion: "20018" uid: 7deba944-68a6-4ac8-8b6c-ee3ccab3f7ca kind: Event lastTimestamp: "2026-05-20T14:10:19Z" message: 'Created pod: raw-sklearn-runtime-83724-predictor-d9995568c-7jh6h' metadata: creationTimestamp: "2026-05-20T14:10:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:10:19Z" name: raw-sklearn-runtime-83724-predictor-d9995568c.18b14b63d72fbbdb namespace: kserve-ci-e2e-test resourceVersion: "20030" uid: a00d85d2-e7e6-4c4c-829b-ca71e3418cdf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:10:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-83724-predictor namespace: kserve-ci-e2e-test resourceVersion: "20016" uid: 035da724-dde2-4c58-a2a1-50fd515d6297 kind: Event lastTimestamp: "2026-05-20T14:10:19Z" message: Scaled up replica set raw-sklearn-runtime-83724-predictor-d9995568c from 0 to 1 metadata: creationTimestamp: "2026-05-20T14:10:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:10:19Z" name: raw-sklearn-runtime-83724-predictor.18b14b63d3d354f3 namespace: kserve-ci-e2e-test resourceVersion: "20020" uid: 4e4cc9b0-8fd2-455f-8898-cbae44e876c8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:10:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-83724-predictor namespace: kserve-ci-e2e-test resourceVersion: "20017" uid: b0388269-be89-4bde-b08e-1f47fbb0bf0d kind: Event lastTimestamp: "2026-05-20T14:10:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:10:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:10:34Z" name: raw-sklearn-runtime-83724-predictor.18b14b6752e51420 namespace: kserve-ci-e2e-test resourceVersion: "20171" uid: 22bda69c-9ecc-44c6-b71e-c8921df80530 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:10:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-83724-predictor namespace: kserve-ci-e2e-test resourceVersion: "20017" uid: b0388269-be89-4bde-b08e-1f47fbb0bf0d kind: Event lastTimestamp: "2026-05-20T14:10:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-20T14:10:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:10:34Z" name: raw-sklearn-runtime-83724-predictor.18b14b67534db7d4 namespace: kserve-ci-e2e-test resourceVersion: "20173" uid: 67c4b14a-3bb7-4885-8709-21137f493919 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:10:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-83724-predictor namespace: kserve-ci-e2e-test resourceVersion: "20172" uid: b0388269-be89-4bde-b08e-1f47fbb0bf0d kind: Event lastTimestamp: "2026-05-20T14:11:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:10:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:11:35Z" name: raw-sklearn-runtime-83724-predictor.18b14b6ad1f4089e namespace: kserve-ci-e2e-test resourceVersion: "20603" uid: 05c083fc-8239-48bb-8275-8d26d3cef7dc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-20T14:10:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-83724-predictor namespace: kserve-ci-e2e-test resourceVersion: "20172" uid: b0388269-be89-4bde-b08e-1f47fbb0bf0d kind: Event lastTimestamp: "2026-05-20T14:11:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-20T14:10:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-20T14:11:35Z" name: raw-sklearn-runtime-83724-predictor.18b14b6ad24e6543 namespace: kserve-ci-e2e-test resourceVersion: "20604" uid: 566417c3-e9f3-4591-8bcc-b77eba9e622c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-05-20T14:10:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-83724 namespace: kserve-ci-e2e-test resourceVersion: "19986" uid: c5340040-9804-496f-9dc5-25b5b52d9188 kind: Event lastTimestamp: "2026-05-20T14:11:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-20T14:10:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:11:31Z" name: raw-sklearn-runtime-83724.18b14b63d3adbc35 namespace: kserve-ci-e2e-test resourceVersion: "20580" uid: a9a527ae-1d31-4c47-ba44-3eb247e9b007 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:10:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-83724 namespace: kserve-ci-e2e-test resourceVersion: "19986" uid: c5340040-9804-496f-9dc5-25b5b52d9188 kind: Event lastTimestamp: "2026-05-20T14:10:19Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-83724": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-83724": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-20T14:10:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:10:19Z" name: raw-sklearn-runtime-83724.18b14b63d4085882 namespace: kserve-ci-e2e-test resourceVersion: "20022" uid: 4156e545-b7d5-494e-a0e0-5b3cb5a76119 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-20T14:11:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-83724 namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: c5340040-9804-496f-9dc5-25b5b52d9188 kind: Event lastTimestamp: "2026-05-20T14:11:31Z" message: InferenceService [raw-sklearn-runtime-83724] is Ready metadata: creationTimestamp: "2026-05-20T14:11:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-20T14:11:31Z" name: raw-sklearn-runtime-83724.18b14b74764bcf1a namespace: kserve-ci-e2e-test resourceVersion: "20578" uid: 43c8c122-9093-4433-a7a4-5b3f126382bd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "20868"