--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:25:16.068401Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-a4d11b-predictor-67b5fcc647-bv5dt namespace: kserve-ci-e2e-test resourceVersion: "19139" uid: fda7d385-7ff2-4f30-91cf-1a1c678a0f25 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-a4d11b-predictor-67b5fcc647-bv5dt to ip-10-0-135-184.ec2.internal metadata: creationTimestamp: "2026-04-20T20:25:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:25:16Z" name: isvc-init-fail-a4d11b-predictor-67b5fcc647-bv5dt.18a82a7097c18e52 namespace: kserve-ci-e2e-test resourceVersion: "19143" uid: 908e0588-baea-4826-bd2a-d3f24a236c8e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5766b54df6-m4bt5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:16Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-a4d11b-predictor-67b5fcc647-bv5dt namespace: kserve-ci-e2e-test resourceVersion: "19147" uid: fda7d385-7ff2-4f30-91cf-1a1c678a0f25 kind: Event lastTimestamp: "2026-04-20T20:25:16Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:25:16Z" name: isvc-init-fail-a4d11b-predictor-67b5fcc647-bv5dt.18a82a70b12acd6a namespace: kserve-ci-e2e-test resourceVersion: "19154" uid: 1d2c8c61-e11f-4b79-b151-69c4742b71ef reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-a4d11b-predictor-67b5fcc647-bv5dt namespace: kserve-ci-e2e-test resourceVersion: "19141" uid: fda7d385-7ff2-4f30-91cf-1a1c678a0f25 kind: Event lastTimestamp: "2026-04-20T20:25:20Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" already present on machine metadata: creationTimestamp: "2026-04-20T20:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:20Z" name: isvc-init-fail-a4d11b-predictor-67b5fcc647-bv5dt.18a82a70b29cb134 namespace: kserve-ci-e2e-test resourceVersion: "19200" uid: 204a3d3d-0da3-44df-9377-04bbf159d29e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-a4d11b-predictor-67b5fcc647-bv5dt namespace: kserve-ci-e2e-test resourceVersion: "19141" uid: fda7d385-7ff2-4f30-91cf-1a1c678a0f25 kind: Event lastTimestamp: "2026-04-20T20:25:20Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-20T20:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:20Z" name: isvc-init-fail-a4d11b-predictor-67b5fcc647-bv5dt.18a82a70b5627c3a namespace: kserve-ci-e2e-test resourceVersion: "19206" uid: 873372f5-76ab-4d40-a75b-d43bff9ba851 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:25:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-a4d11b-predictor-67b5fcc647-bv5dt namespace: kserve-ci-e2e-test resourceVersion: "19141" uid: fda7d385-7ff2-4f30-91cf-1a1c678a0f25 kind: Event lastTimestamp: "2026-04-20T20:25:20Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-20T20:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:20Z" name: isvc-init-fail-a4d11b-predictor-67b5fcc647-bv5dt.18a82a70b5f7cdba namespace: kserve-ci-e2e-test resourceVersion: "19207" uid: 6ea0da7b-9344-4d42-9d37-c9450a460a36 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-a4d11b-predictor-67b5fcc647-bv5dt namespace: kserve-ci-e2e-test resourceVersion: "19141" uid: fda7d385-7ff2-4f30-91cf-1a1c678a0f25 kind: Event lastTimestamp: "2026-04-20T20:25:21Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-04-20T20:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:21Z" name: isvc-init-fail-a4d11b-predictor-67b5fcc647-bv5dt.18a82a71e14aa6a3 namespace: kserve-ci-e2e-test resourceVersion: "19273" uid: a8b81a08-0a92-45cc-a6bb-8dda98ef04ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-a4d11b-predictor-67b5fcc647 namespace: kserve-ci-e2e-test resourceVersion: "19130" uid: db89a9d7-3f58-4e89-aeb9-cdbf23283f21 kind: Event lastTimestamp: "2026-04-20T20:25:16Z" message: 'Created pod: isvc-init-fail-a4d11b-predictor-67b5fcc647-bv5dt' metadata: creationTimestamp: "2026-04-20T20:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:16Z" name: isvc-init-fail-a4d11b-predictor-67b5fcc647.18a82a70975dda4f namespace: kserve-ci-e2e-test resourceVersion: "19140" uid: 7b5557b0-7a6e-4cf3-a882-d8d6bc1ec017 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-a4d11b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19128" uid: 35d6d5ab-c4ac-4f77-886d-1f80ebea5a33 kind: Event lastTimestamp: "2026-04-20T20:25:16Z" message: Scaled up replica set isvc-init-fail-a4d11b-predictor-67b5fcc647 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:16Z" name: isvc-init-fail-a4d11b-predictor.18a82a70946c3c50 namespace: kserve-ci-e2e-test resourceVersion: "19132" uid: 393155de-1b07-4451-8013-b631a51a8935 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-20T20:25:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-a4d11b namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: 92ad8773-2b75-4b1b-b42a-823ae2671a31 kind: Event lastTimestamp: "2026-04-20T20:25:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:20Z" name: isvc-init-fail-a4d11b.18a82a70944c403a namespace: kserve-ci-e2e-test resourceVersion: "19208" uid: 0413d129-0aee-4ba4-873f-bea28e555905 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-a4d11b namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: 92ad8773-2b75-4b1b-b42a-823ae2671a31 kind: Event lastTimestamp: "2026-04-20T20:25:16Z" message: 'Failed to update status for InferenceService "isvc-init-fail-a4d11b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-a4d11b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:16Z" name: isvc-init-fail-a4d11b.18a82a7094a087a4 namespace: kserve-ci-e2e-test resourceVersion: "19134" uid: 8dfceb35-a8a6-446f-851b-373a944e1203 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-a4d11b namespace: kserve-ci-e2e-test resourceVersion: "19101" uid: 92ad8773-2b75-4b1b-b42a-823ae2671a31 kind: Event lastTimestamp: "2026-04-20T20:25:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-a4d11b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:25:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:16Z" name: isvc-init-fail-a4d11b.18a82a7094a0cd71 namespace: kserve-ci-e2e-test resourceVersion: "19135" uid: 37592c12-70ce-4d6c-bcb4-0f60a225a498 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:19:56.245854Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16479" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb to ip-10-0-135-184.ec2.internal metadata: creationTimestamp: "2026-04-20T20:19:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:19:56Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a2620d8cd6e namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: ee195975-0624-4ec9-8fb6-bf7c0beb8cee reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5766b54df6-m4bt5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: "2026-04-20T20:19:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-raw-5be0c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:19:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:56Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a262c79a9c5 namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: 232b75d5-4072-44ee-a359-df25dee58acc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16487" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: "2026-04-20T20:19:57Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:19:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:19:57Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a265db6e09f namespace: kserve-ci-e2e-test resourceVersion: "16501" uid: db29f53d-996f-4b99-bebc-33c07bca4316 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: "2026-04-20T20:19:57Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" already present on machine metadata: creationTimestamp: "2026-04-20T20:19:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:57Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a265f40bd4e namespace: kserve-ci-e2e-test resourceVersion: "16503" uid: 81a7ac6c-597a-40be-ac8d-4760fea7f15a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: "2026-04-20T20:19:57Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-20T20:19:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:57Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a26622b28ef namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: 2bf3113b-ad21-459d-9365-6b0d4244559f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: "2026-04-20T20:19:57Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-20T20:19:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:57Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a2662a0315e namespace: kserve-ci-e2e-test resourceVersion: "16507" uid: b3577862-dd4e-48e5-a646-178743d18ef5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:20:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: "2026-04-20T20:20:01Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:20:01Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a2759562166 namespace: kserve-ci-e2e-test resourceVersion: "16540" uid: 3f2d7315-6cb0-4652-b290-71107cd47834 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:20:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: "2026-04-20T20:20:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:20:01Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a275ca73e0e namespace: kserve-ci-e2e-test resourceVersion: "16544" uid: 6c806358-2ef0-4982-9d9a-f2667736d0f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:20:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: "2026-04-20T20:20:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:20:01Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a275d205230 namespace: kserve-ci-e2e-test resourceVersion: "16545" uid: 66d0f6dd-2db9-4af5-95bf-9b59795574a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:20:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: "2026-04-20T20:20:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:20:01Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a275d2d7311 namespace: kserve-ci-e2e-test resourceVersion: "16546" uid: 2a1dec5c-b64a-46dd-aa6d-1b8b75db020c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:20:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: "2026-04-20T20:20:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:20:01Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a27618aa819 namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 363bbb53-f335-4dca-8507-c07e27fc3449 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:20:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: "2026-04-20T20:20:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:20:01Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a276201ce18 namespace: kserve-ci-e2e-test resourceVersion: "16549" uid: 0798991b-2ba0-4098-bef1-e749675c30d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:20:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: "2026-04-20T20:20:01Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:04387bc02593a57a59970799a98bb0f0c4351f10f0912ba6f376f88063c848eb" already present on machine metadata: creationTimestamp: "2026-04-20T20:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:20:01Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a27620f7504 namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: 5ba6643b-8939-40d4-aca2-d0e9290e55b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:20:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: "2026-04-20T20:20:01Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-20T20:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:20:01Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a27650f3dbd namespace: kserve-ci-e2e-test resourceVersion: "16551" uid: 00e91ada-7d71-47c0-87d6-221668abac4e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:20:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: "2026-04-20T20:20:01Z" message: Started container agent metadata: creationTimestamp: "2026-04-20T20:20:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:20:01Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a2765867108 namespace: kserve-ci-e2e-test resourceVersion: "16552" uid: b9aa6047-51d3-47c9-a109-19c2ec4e9b3b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-20T20:20:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: "2026-04-20T20:21:39Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:20:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:21:39Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a27955d2c60 namespace: kserve-ci-e2e-test resourceVersion: "17387" uid: 83750c6d-9008-4620-96a0-b6349f69a0d9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-20T20:20:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: "2026-04-20T20:21:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-20T20:20:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:21:29Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a27d11dbf6f namespace: kserve-ci-e2e-test resourceVersion: "17305" uid: f94fd9dd-ffeb-4fb8-abe8-886f0d3553d8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:21:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: "2026-04-20T20:21:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:21:21Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a39f75b2d16 namespace: kserve-ci-e2e-test resourceVersion: "17185" uid: f65d0aa3-c820-4098-b2f9-37689014c5d8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:21:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: "2026-04-20T20:21:21Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-20T20:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:21:21Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a39f75e23ad namespace: kserve-ci-e2e-test resourceVersion: "17186" uid: a87cea3d-2253-49cf-acef-2b7b0f033547 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:21:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: "2026-04-20T20:21:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:21:21Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a39f7610866 namespace: kserve-ci-e2e-test resourceVersion: "17187" uid: cdda6df7-7741-4147-b7ea-8239dbf51792 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:21:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 5aafa7c5-2a5a-4436-96c0-5528d9796f1e kind: Event lastTimestamp: "2026-04-20T20:21:39Z" message: 'Readiness probe failed: Get "https://10.133.0.38:8643/healthz": dial tcp 10.133.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:21:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:21:39Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb.18a82a3aad41a103 namespace: kserve-ci-e2e-test resourceVersion: "17386" uid: 93a16c22-0798-4fb9-a133-42bb020977a3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-5be0c-predictor-7c47c997b7 namespace: kserve-ci-e2e-test resourceVersion: "16468" uid: b32cf229-5e41-4e0c-bf1f-93eda28d6555 kind: Event lastTimestamp: "2026-04-20T20:19:56Z" message: 'Created pod: isvc-logger-raw-5be0c-predictor-7c47c997b7-8k9kb' metadata: creationTimestamp: "2026-04-20T20:19:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:19:56Z" name: isvc-logger-raw-5be0c-predictor-7c47c997b7.18a82a2620757c84 namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: 1d744ac4-d348-4ab8-91d6-cfd4b8d38fba reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-5be0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16466" uid: 6451ac79-7d2e-4357-a80e-64edc82e849e kind: Event lastTimestamp: "2026-04-20T20:19:56Z" message: Scaled up replica set isvc-logger-raw-5be0c-predictor-7c47c997b7 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:19:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:19:56Z" name: isvc-logger-raw-5be0c-predictor.18a82a261ce2cb9a namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 5989a99b-d224-4b03-88b4-0669fdaa810c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:20:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-5be0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: 5a9c9b37-8c5d-47ae-8799-4c2cef18fe3f kind: Event lastTimestamp: "2026-04-20T20:20:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:20:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:20:41Z" name: isvc-logger-raw-5be0c-predictor.18a82a299c88ffca namespace: kserve-ci-e2e-test resourceVersion: "16851" uid: 0fbe6440-85f1-4765-add4-5b4fb884f87c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:20:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-5be0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: 5a9c9b37-8c5d-47ae-8799-4c2cef18fe3f kind: Event lastTimestamp: "2026-04-20T20:20:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:20:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:20:41Z" name: isvc-logger-raw-5be0c-predictor.18a82a299d53152e namespace: kserve-ci-e2e-test resourceVersion: "16852" uid: 6b3bec98-c335-4d0e-a5fd-e73f332ce659 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:20:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-5be0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16637" uid: 5a9c9b37-8c5d-47ae-8799-4c2cef18fe3f kind: Event lastTimestamp: "2026-04-20T20:21:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:21:11Z" name: isvc-logger-raw-5be0c-predictor.18a82a3419f61e4a namespace: kserve-ci-e2e-test resourceVersion: "17073" uid: 16eff2ef-b698-4df5-802c-2d45e0bec64c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:20:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-5be0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16637" uid: 5a9c9b37-8c5d-47ae-8799-4c2cef18fe3f kind: Event lastTimestamp: "2026-04-20T20:21:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:21:11Z" name: isvc-logger-raw-5be0c-predictor.18a82a341a70e73f namespace: kserve-ci-e2e-test resourceVersion: "17074" uid: 51ad8d81-ef34-4e79-9b6a-60dc33a44a85 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-20T20:19:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-5be0c namespace: kserve-ci-e2e-test resourceVersion: "16456" uid: 7c7691b6-e04c-4d01-bbdc-3e70bc4ceceb kind: Event lastTimestamp: "2026-04-20T20:21:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:19:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:21:09Z" name: isvc-logger-raw-5be0c.18a82a261cb6ad5d namespace: kserve-ci-e2e-test resourceVersion: "17063" uid: 25274c60-c4e1-4bc5-82d5-df8b7a74f3f4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-5be0c namespace: kserve-ci-e2e-test resourceVersion: "16456" uid: 7c7691b6-e04c-4d01-bbdc-3e70bc4ceceb kind: Event lastTimestamp: "2026-04-20T20:19:56Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-5be0c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-5be0c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:19:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:19:56Z" name: isvc-logger-raw-5be0c.18a82a261d3d6fc0 namespace: kserve-ci-e2e-test resourceVersion: "16473" uid: 8a78482e-a5a0-4c6b-8153-4c23b0288469 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:21:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-5be0c namespace: kserve-ci-e2e-test resourceVersion: "17061" uid: 7c7691b6-e04c-4d01-bbdc-3e70bc4ceceb kind: Event lastTimestamp: "2026-04-20T20:21:09Z" message: InferenceService [isvc-logger-raw-5be0c] is Ready metadata: creationTimestamp: "2026-04-20T20:21:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:21:09Z" name: isvc-logger-raw-5be0c.18a82a3735d86602 namespace: kserve-ci-e2e-test resourceVersion: "17062" uid: 99de6f16-6c37-4afc-9a4f-3864127221f3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:23:41.760383Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj namespace: kserve-ci-e2e-test resourceVersion: "18295" uid: a7af65ce-7427-4518-b4d1-39db352355fb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj to ip-10-0-135-184.ec2.internal metadata: creationTimestamp: "2026-04-20T20:23:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:23:41Z" name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj.18a82a5aa28f7033 namespace: kserve-ci-e2e-test resourceVersion: "18300" uid: 5973a855-10dc-4907-be2a-8c898a4acef0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5766b54df6-m4bt5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:23:41Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj namespace: kserve-ci-e2e-test resourceVersion: "18298" uid: a7af65ce-7427-4518-b4d1-39db352355fb kind: Event lastTimestamp: "2026-04-20T20:23:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-primary-7f229f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:23:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:23:41Z" name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj.18a82a5aac828e5a namespace: kserve-ci-e2e-test resourceVersion: "18307" uid: 74782e1a-af8b-46e8-af84-d3e1c2fc82a7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:23:42Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj namespace: kserve-ci-e2e-test resourceVersion: "18305" uid: a7af65ce-7427-4518-b4d1-39db352355fb kind: Event lastTimestamp: "2026-04-20T20:23:42Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:23:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:23:42Z" name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj.18a82a5adfbb88e0 namespace: kserve-ci-e2e-test resourceVersion: "18319" uid: 582bea3c-915e-4eaf-bbdb-efbcf9ac1bff reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:23:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj namespace: kserve-ci-e2e-test resourceVersion: "18298" uid: a7af65ce-7427-4518-b4d1-39db352355fb kind: Event lastTimestamp: "2026-04-20T20:23:42Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" already present on machine metadata: creationTimestamp: "2026-04-20T20:23:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:23:42Z" name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj.18a82a5ae113b2c5 namespace: kserve-ci-e2e-test resourceVersion: "18321" uid: f45e56fb-8862-403e-bdb3-27b19480c59b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:23:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj namespace: kserve-ci-e2e-test resourceVersion: "18298" uid: a7af65ce-7427-4518-b4d1-39db352355fb kind: Event lastTimestamp: "2026-04-20T20:23:42Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-20T20:23:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:23:42Z" name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj.18a82a5ae440b380 namespace: kserve-ci-e2e-test resourceVersion: "18322" uid: d1012826-deab-409b-b48d-dfa01b818aaa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:23:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj namespace: kserve-ci-e2e-test resourceVersion: "18298" uid: a7af65ce-7427-4518-b4d1-39db352355fb kind: Event lastTimestamp: "2026-04-20T20:23:42Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-20T20:23:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:23:42Z" name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj.18a82a5ae4b9d130 namespace: kserve-ci-e2e-test resourceVersion: "18323" uid: 566f255c-c7d7-4bc4-b28f-f2410f2d3365 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:23:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj namespace: kserve-ci-e2e-test resourceVersion: "18298" uid: a7af65ce-7427-4518-b4d1-39db352355fb kind: Event lastTimestamp: "2026-04-20T20:23:47Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:23:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:23:47Z" name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj.18a82a5bea2237ec namespace: kserve-ci-e2e-test resourceVersion: "18359" uid: c2990e40-75d6-45dd-a2e6-67f17c4334e6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:23:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj namespace: kserve-ci-e2e-test resourceVersion: "18298" uid: a7af65ce-7427-4518-b4d1-39db352355fb kind: Event lastTimestamp: "2026-04-20T20:23:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:23:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:23:47Z" name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj.18a82a5bed38c90c namespace: kserve-ci-e2e-test resourceVersion: "18363" uid: b84d2d01-af1b-4a11-bbc8-f8ce8bf1e9b9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:23:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj namespace: kserve-ci-e2e-test resourceVersion: "18298" uid: a7af65ce-7427-4518-b4d1-39db352355fb kind: Event lastTimestamp: "2026-04-20T20:23:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:23:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:23:47Z" name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj.18a82a5bedb81981 namespace: kserve-ci-e2e-test resourceVersion: "18364" uid: 6fef81a9-8a56-4724-948a-6fb027d91e6c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:23:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj namespace: kserve-ci-e2e-test resourceVersion: "18298" uid: a7af65ce-7427-4518-b4d1-39db352355fb kind: Event lastTimestamp: "2026-04-20T20:23:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:23:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:23:47Z" name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj.18a82a5bedc62919 namespace: kserve-ci-e2e-test resourceVersion: "18366" uid: 6e48bdd5-15b3-405e-95d9-7e166ef9b975 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:23:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj namespace: kserve-ci-e2e-test resourceVersion: "18298" uid: a7af65ce-7427-4518-b4d1-39db352355fb kind: Event lastTimestamp: "2026-04-20T20:23:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:23:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:23:47Z" name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj.18a82a5bf10b3dd4 namespace: kserve-ci-e2e-test resourceVersion: "18369" uid: 956da1a5-17d3-477e-abed-ec81024cad67 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:23:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj namespace: kserve-ci-e2e-test resourceVersion: "18298" uid: a7af65ce-7427-4518-b4d1-39db352355fb kind: Event lastTimestamp: "2026-04-20T20:23:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:23:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:23:47Z" name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj.18a82a5bf1838e5d namespace: kserve-ci-e2e-test resourceVersion: "18370" uid: a2c051db-f6df-4eec-9d9a-9ed473a26115 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-20T20:23:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj namespace: kserve-ci-e2e-test resourceVersion: "18298" uid: a7af65ce-7427-4518-b4d1-39db352355fb kind: Event lastTimestamp: "2026-04-20T20:24:44Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:23:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:24:44Z" name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj.18a82a5c2616f246 namespace: kserve-ci-e2e-test resourceVersion: "18780" uid: 63e27af5-c88d-4b71-9e0b-6fc1b8ae6845 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj namespace: kserve-ci-e2e-test resourceVersion: "18298" uid: a7af65ce-7427-4518-b4d1-39db352355fb kind: Event lastTimestamp: "2026-04-20T20:25:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:15Z" name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj.18a82a70929ca06a namespace: kserve-ci-e2e-test resourceVersion: "19120" uid: b5107d31-377a-472e-80a0-60a6100221ba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj namespace: kserve-ci-e2e-test resourceVersion: "18298" uid: a7af65ce-7427-4518-b4d1-39db352355fb kind: Event lastTimestamp: "2026-04-20T20:25:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:25:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:15Z" name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj.18a82a70929d400f namespace: kserve-ci-e2e-test resourceVersion: "19123" uid: ba75e4f4-966c-4012-8818-8a304645b76d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj namespace: kserve-ci-e2e-test resourceVersion: "18298" uid: a7af65ce-7427-4518-b4d1-39db352355fb kind: Event lastTimestamp: "2026-04-20T20:25:19Z" message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": dial tcp 10.133.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:25:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:19Z" name: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj.18a82a71563e8999 namespace: kserve-ci-e2e-test resourceVersion: "19187" uid: beba962d-3302-484a-97d4-70a2e83a3b93 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:23:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-7f229f-predictor-7bd7856b4d namespace: kserve-ci-e2e-test resourceVersion: "18284" uid: f18f0ff5-e22a-47ba-a0a0-c22eb6e52ac9 kind: Event lastTimestamp: "2026-04-20T20:23:41Z" message: 'Created pod: isvc-primary-7f229f-predictor-7bd7856b4d-q5sgj' metadata: creationTimestamp: "2026-04-20T20:23:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:23:41Z" name: isvc-primary-7f229f-predictor-7bd7856b4d.18a82a5aa22b08fd namespace: kserve-ci-e2e-test resourceVersion: "18297" uid: f1a137b2-2cce-41c7-aa50-a42361c43e22 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:23:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-7f229f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18281" uid: 1660e501-510e-4427-9852-36e686eb16f9 kind: Event lastTimestamp: "2026-04-20T20:23:41Z" message: Scaled up replica set isvc-primary-7f229f-predictor-7bd7856b4d from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:23:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:23:41Z" name: isvc-primary-7f229f-predictor.18a82a5a9d882280 namespace: kserve-ci-e2e-test resourceVersion: "18286" uid: 5b858eb2-82eb-4278-a95a-eef4ca0a9763 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:23:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-7f229f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18282" uid: 01e1ac3a-efba-4bfa-afa0-0b04145ef360 kind: Event lastTimestamp: "2026-04-20T20:24:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:23:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:24:26Z" name: isvc-primary-7f229f-predictor.18a82a5e1c08803b namespace: kserve-ci-e2e-test resourceVersion: "18652" uid: f3269968-d9c8-40e1-949d-ef0687dd2daa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:23:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-7f229f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18282" uid: 01e1ac3a-efba-4bfa-afa0-0b04145ef360 kind: Event lastTimestamp: "2026-04-20T20:24:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:23:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:24:26Z" name: isvc-primary-7f229f-predictor.18a82a5e1c6a07dc namespace: kserve-ci-e2e-test resourceVersion: "18653" uid: fc3ffa1e-cde5-400d-aa44-82c72117dd3c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:24:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-7f229f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18443" uid: 01e1ac3a-efba-4bfa-afa0-0b04145ef360 kind: Event lastTimestamp: "2026-04-20T20:25:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:11Z" name: isvc-primary-7f229f-predictor.18a82a689976cebe namespace: kserve-ci-e2e-test resourceVersion: "19046" uid: 058d1d19-f9c6-4453-99b0-f25392571a65 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:24:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-7f229f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18443" uid: 01e1ac3a-efba-4bfa-afa0-0b04145ef360 kind: Event lastTimestamp: "2026-04-20T20:25:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:11Z" name: isvc-primary-7f229f-predictor.18a82a6899d5aa09 namespace: kserve-ci-e2e-test resourceVersion: "19047" uid: ff708d5f-dd35-4a59-8199-65fbc80cde01 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-04-20T20:23:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-7f229f namespace: kserve-ci-e2e-test resourceVersion: "18255" uid: 41ae4299-2eda-443b-a31e-49c5a08b86b8 kind: Event lastTimestamp: "2026-04-20T20:24:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:23:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:24:54Z" name: isvc-primary-7f229f.18a82a5a9d63106d namespace: kserve-ci-e2e-test resourceVersion: "18856" uid: ff4e888a-18fc-4f9b-901c-715956a49fb9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:23:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-7f229f namespace: kserve-ci-e2e-test resourceVersion: "18255" uid: 41ae4299-2eda-443b-a31e-49c5a08b86b8 kind: Event lastTimestamp: "2026-04-20T20:23:41Z" message: 'Failed to update status for InferenceService "isvc-primary-7f229f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-7f229f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:23:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:23:41Z" name: isvc-primary-7f229f.18a82a5a9db1593b namespace: kserve-ci-e2e-test resourceVersion: "18289" uid: 76f94826-4d87-47c6-b397-bce3efbb1cd9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:23:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-7f229f namespace: kserve-ci-e2e-test resourceVersion: "18255" uid: 41ae4299-2eda-443b-a31e-49c5a08b86b8 kind: Event lastTimestamp: "2026-04-20T20:23:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-7f229f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:23:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:23:41Z" name: isvc-primary-7f229f.18a82a5a9db1c361 namespace: kserve-ci-e2e-test resourceVersion: "18290" uid: 7b8514d8-35e2-41b8-a9a0-a163f52017e7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:24:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-7f229f namespace: kserve-ci-e2e-test resourceVersion: "18854" uid: 41ae4299-2eda-443b-a31e-49c5a08b86b8 kind: Event lastTimestamp: "2026-04-20T20:24:54Z" message: InferenceService [isvc-primary-7f229f] is Ready metadata: creationTimestamp: "2026-04-20T20:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:24:54Z" name: isvc-primary-7f229f.18a82a6b88e1654d namespace: kserve-ci-e2e-test resourceVersion: "18855" uid: 467f4230-832a-4d55-9cfd-5b734731e8c3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:14:20.375866Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13389" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 to ip-10-0-135-184.ec2.internal metadata: creationTimestamp: "2026-04-20T20:14:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:14:20Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829d7ed6f95cc namespace: kserve-ci-e2e-test resourceVersion: "13394" uid: 337b28de-4a1a-4a5d-b6f5-61096f1e1bde reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5766b54df6-m4bt5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13399" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:14:20Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:14:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:14:20Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829d80711dbe4 namespace: kserve-ci-e2e-test resourceVersion: "13412" uid: 83f7a4fd-9aa0-473b-9471-1529df329aea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:14:20Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" metadata: creationTimestamp: "2026-04-20T20:14:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:20Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829d80890fa24 namespace: kserve-ci-e2e-test resourceVersion: "13414" uid: b53661e7-bfbb-49be-9032-4a2951d0d5f4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:14:23Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" in 2.869s (2.869s including waiting). Image size: 300951977 bytes.' metadata: creationTimestamp: "2026-04-20T20:14:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:23Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829d8b3946c94 namespace: kserve-ci-e2e-test resourceVersion: "13435" uid: 35346148-992e-4913-b9f0-0db5f1dad421 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:14:23Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-20T20:14:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:23Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829d8b6cc1f26 namespace: kserve-ci-e2e-test resourceVersion: "13437" uid: c0eddf0a-d53a-4849-9c71-6984c140e392 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:23Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:14:23Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-20T20:14:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:23Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829d8b7575743 namespace: kserve-ci-e2e-test resourceVersion: "13438" uid: e809ab46-e792-475e-8b7b-576b6e08a87c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:14:28Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" metadata: creationTimestamp: "2026-04-20T20:14:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:28Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829d9c34692ef namespace: kserve-ci-e2e-test resourceVersion: "13479" uid: bbd3da92-f9bc-418a-ae9b-b7af2dee740d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:14:41Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" in 13.01s (13.01s including waiting). Image size: 1560610732 bytes.' metadata: creationTimestamp: "2026-04-20T20:14:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:41Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829dccabd5d2f namespace: kserve-ci-e2e-test resourceVersion: "13575" uid: b875e2ad-cb37-42bc-8d7d-9db4804a1a1a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:14:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:14:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:41Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829dcce4ab661 namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: 0a4f564e-8822-4a32-b7c1-66f7b8143f84 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:14:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:14:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:41Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829dccec568a2 namespace: kserve-ci-e2e-test resourceVersion: "13578" uid: 9c6e7a8d-ea68-42cc-8b0c-d913e368da6f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:14:41Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-20T20:14:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:41Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829dcced3c447 namespace: kserve-ci-e2e-test resourceVersion: "13579" uid: 835057d5-fcd4-49ce-9dce-cb33282d1a59 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:14:43Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.178s (2.178s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-20T20:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:43Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829dd50b32c9d namespace: kserve-ci-e2e-test resourceVersion: "13593" uid: acc459c7-e81c-4886-a853-ab22872b30a8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:14:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:43Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829dd54054882 namespace: kserve-ci-e2e-test resourceVersion: "13594" uid: 9d26dc77-2560-471f-b179-6972a4496327 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:14:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:43Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829dd54822374 namespace: kserve-ci-e2e-test resourceVersion: "13595" uid: 31fab658-3168-4710-9d99-5570c7d1a328 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:14:43Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:04387bc02593a57a59970799a98bb0f0c4351f10f0912ba6f376f88063c848eb" metadata: creationTimestamp: "2026-04-20T20:14:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:43Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829dd549000b1 namespace: kserve-ci-e2e-test resourceVersion: "13596" uid: c8da5426-ca15-4672-bcd8-4cc8190814c0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:14:46Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:04387bc02593a57a59970799a98bb0f0c4351f10f0912ba6f376f88063c848eb" in 2.958s (2.958s including waiting). Image size: 237897292 bytes.' metadata: creationTimestamp: "2026-04-20T20:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:46Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829de04e4c6d4 namespace: kserve-ci-e2e-test resourceVersion: "13621" uid: c1db6961-7baf-4764-977a-b76f1c3b41d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:14:46Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-04-20T20:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:46Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829de0804a1cd namespace: kserve-ci-e2e-test resourceVersion: "13622" uid: 286fa2f2-4cc1-4bd6-938f-2a355394e684 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:14:46Z" message: Started container agent metadata: creationTimestamp: "2026-04-20T20:14:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:14:46Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829de087eb172 namespace: kserve-ci-e2e-test resourceVersion: "13623" uid: 148428fa-5fa2-47d2-9e4b-d35200919aa3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-20T20:14:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:16:20Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:14:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:20Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829de716e5d3f namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 5e32186b-86ab-441d-a22b-0c0f7586d190 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-20T20:14:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:16:20Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-20T20:14:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:20Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829de71789a1f namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: 3504fc6e-8386-4bb6-ba49-0e2fc0c9e366 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:16:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:05Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829f06921c55e namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 4384e7c5-479f-4025-ada2-4d01bff4c99a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:16:05Z" message: Stopping container agent metadata: creationTimestamp: "2026-04-20T20:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:05Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829f06921e604 namespace: kserve-ci-e2e-test resourceVersion: "14225" uid: f2b28772-a09f-4240-895b-29d1e07dcdee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:16:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:05Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829f06922565f namespace: kserve-ci-e2e-test resourceVersion: "14227" uid: b408a210-ceac-4d4a-badb-13f54282d15a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-20T20:16:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8 namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: 75b0814e-b821-4519-974f-33493a091ca5 kind: Event lastTimestamp: "2026-04-20T20:16:28Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:16:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:28Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8.18a829f111c71885 namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: 47850fc4-fd9e-4d57-9948-258ea633163c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f namespace: kserve-ci-e2e-test resourceVersion: "13382" uid: 796f890d-c1ff-4a8c-910a-46ebf3cd4dd8 kind: Event lastTimestamp: "2026-04-20T20:14:20Z" message: 'Created pod: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f-65rl8' metadata: creationTimestamp: "2026-04-20T20:14:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:14:20Z" name: isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f.18a829d7ecfffe19 namespace: kserve-ci-e2e-test resourceVersion: "13390" uid: fd36df97-a62d-446b-885b-2e75ee472c44 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-8ea43-predictor namespace: kserve-ci-e2e-test resourceVersion: "13381" uid: 7a881c23-eb00-46a7-a714-c38e57356d0c kind: Event lastTimestamp: "2026-04-20T20:14:20Z" message: Scaled up replica set isvc-raw-sklearn-batcher-8ea43-predictor-64b4f7648f from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:14:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:14:20Z" name: isvc-raw-sklearn-batcher-8ea43-predictor.18a829d7e7805b55 namespace: kserve-ci-e2e-test resourceVersion: "13383" uid: fca7d4d2-a671-4eda-b96f-0388b9dae682 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:14:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-8ea43-predictor namespace: kserve-ci-e2e-test resourceVersion: "13393" uid: aaf38d8b-232a-4e77-a17d-19db59fa71d0 kind: Event lastTimestamp: "2026-04-20T20:15:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:14:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:05Z" name: isvc-raw-sklearn-batcher-8ea43-predictor.18a829db6cdd51d6 namespace: kserve-ci-e2e-test resourceVersion: "13762" uid: 9c6ff31f-7df2-4d8c-a78f-2c92a0d0f380 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:14:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-8ea43-predictor namespace: kserve-ci-e2e-test resourceVersion: "13393" uid: aaf38d8b-232a-4e77-a17d-19db59fa71d0 kind: Event lastTimestamp: "2026-04-20T20:15:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:14:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:05Z" name: isvc-raw-sklearn-batcher-8ea43-predictor.18a829db6d484eeb namespace: kserve-ci-e2e-test resourceVersion: "13763" uid: 4d788080-17f5-4a84-9333-5698972bc27c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:15:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-8ea43-predictor namespace: kserve-ci-e2e-test resourceVersion: "13531" uid: aaf38d8b-232a-4e77-a17d-19db59fa71d0 kind: Event lastTimestamp: "2026-04-20T20:15:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:15:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:50Z" name: isvc-raw-sklearn-batcher-8ea43-predictor.18a829e5eb448c14 namespace: kserve-ci-e2e-test resourceVersion: "14093" uid: 7ef8cd61-bf1c-4209-9f1a-d6bdd13b551c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:15:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-8ea43-predictor namespace: kserve-ci-e2e-test resourceVersion: "13531" uid: aaf38d8b-232a-4e77-a17d-19db59fa71d0 kind: Event lastTimestamp: "2026-04-20T20:15:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:15:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:15:50Z" name: isvc-raw-sklearn-batcher-8ea43-predictor.18a829e5ebe0cc47 namespace: kserve-ci-e2e-test resourceVersion: "14094" uid: 17fb8c79-2d5f-4218-87f7-ada40ddc6487 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-04-20T20:14:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-8ea43 namespace: kserve-ci-e2e-test resourceVersion: "13369" uid: 7595f754-f10e-40d6-9752-3bf3eb8fa986 kind: Event lastTimestamp: "2026-04-20T20:15:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:14:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:15:50Z" name: isvc-raw-sklearn-batcher-8ea43.18a829d7edda2cd3 namespace: kserve-ci-e2e-test resourceVersion: "14095" uid: 72bcbf2d-ff64-4076-8720-73246df55c34 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-8ea43 namespace: kserve-ci-e2e-test resourceVersion: "13369" uid: 7595f754-f10e-40d6-9752-3bf3eb8fa986 kind: Event lastTimestamp: "2026-04-20T20:14:20Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-8ea43": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-8ea43": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:14:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:14:20Z" name: isvc-raw-sklearn-batcher-8ea43.18a829d7ee79bd0f namespace: kserve-ci-e2e-test resourceVersion: "13398" uid: cfe168fc-7139-4407-9064-3aa6b0399685 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:14:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-8ea43 namespace: kserve-ci-e2e-test resourceVersion: "13369" uid: 7595f754-f10e-40d6-9752-3bf3eb8fa986 kind: Event lastTimestamp: "2026-04-20T20:14:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-8ea43": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:14:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:14:20Z" name: isvc-raw-sklearn-batcher-8ea43.18a829d7ee7a31fa namespace: kserve-ci-e2e-test resourceVersion: "13401" uid: 99c268a5-b242-4260-afd3-ba130444e7bc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:15:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-8ea43 namespace: kserve-ci-e2e-test resourceVersion: "14090" uid: 7595f754-f10e-40d6-9752-3bf3eb8fa986 kind: Event lastTimestamp: "2026-04-20T20:15:50Z" message: InferenceService [isvc-raw-sklearn-batcher-8ea43] is Ready metadata: creationTimestamp: "2026-04-20T20:15:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:15:50Z" name: isvc-raw-sklearn-batcher-8ea43.18a829ecec68a54b namespace: kserve-ci-e2e-test resourceVersion: "14092" uid: c82e1a90-5e9c-4059-9737-3a065755b32e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:25:01.884087Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-7f229f-predictor-6cb8c497f5-bcglg namespace: kserve-ci-e2e-test resourceVersion: "18941" uid: 6f731566-8fe5-4ec1-8d1d-e8fa572f4efc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-7f229f-predictor-6cb8c497f5-bcglg to ip-10-0-135-184.ec2.internal metadata: creationTimestamp: "2026-04-20T20:25:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:25:01Z" name: isvc-secondary-7f229f-predictor-6cb8c497f5-bcglg.18a82a6d4a4e1604 namespace: kserve-ci-e2e-test resourceVersion: "18947" uid: d2d18cf7-8782-423e-ad9c-c530dfc5348c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5766b54df6-m4bt5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-7f229f-predictor-6cb8c497f5-bcglg namespace: kserve-ci-e2e-test resourceVersion: "18951" uid: 6f731566-8fe5-4ec1-8d1d-e8fa572f4efc kind: Event lastTimestamp: "2026-04-20T20:25:02Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:25:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:25:02Z" name: isvc-secondary-7f229f-predictor-6cb8c497f5-bcglg.18a82a6d63a576f4 namespace: kserve-ci-e2e-test resourceVersion: "18957" uid: 1f802652-eb9b-422d-ba3c-3c911ca79457 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:25:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-7f229f-predictor-6cb8c497f5-bcglg namespace: kserve-ci-e2e-test resourceVersion: "18943" uid: 6f731566-8fe5-4ec1-8d1d-e8fa572f4efc kind: Event lastTimestamp: "2026-04-20T20:25:05Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" already present on machine metadata: creationTimestamp: "2026-04-20T20:25:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:05Z" name: isvc-secondary-7f229f-predictor-6cb8c497f5-bcglg.18a82a6d65041746 namespace: kserve-ci-e2e-test resourceVersion: "18988" uid: 0167e1ef-3cc6-48e0-b436-4519933698eb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:25:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-7f229f-predictor-6cb8c497f5-bcglg namespace: kserve-ci-e2e-test resourceVersion: "18943" uid: 6f731566-8fe5-4ec1-8d1d-e8fa572f4efc kind: Event lastTimestamp: "2026-04-20T20:25:05Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-20T20:25:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:05Z" name: isvc-secondary-7f229f-predictor-6cb8c497f5-bcglg.18a82a6d67a4c289 namespace: kserve-ci-e2e-test resourceVersion: "18992" uid: 17d16b49-c14b-4de9-a6a0-65dd286d4be5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:25:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-7f229f-predictor-6cb8c497f5-bcglg namespace: kserve-ci-e2e-test resourceVersion: "18943" uid: 6f731566-8fe5-4ec1-8d1d-e8fa572f4efc kind: Event lastTimestamp: "2026-04-20T20:25:05Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-20T20:25:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:05Z" name: isvc-secondary-7f229f-predictor-6cb8c497f5-bcglg.18a82a6d681d6b0b namespace: kserve-ci-e2e-test resourceVersion: "18993" uid: 22ac3610-c56b-4701-adc4-92c2c57e02ec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-7f229f-predictor-6cb8c497f5-bcglg namespace: kserve-ci-e2e-test resourceVersion: "18943" uid: 6f731566-8fe5-4ec1-8d1d-e8fa572f4efc kind: Event lastTimestamp: "2026-04-20T20:25:09Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-7f229f-predictor-6cb8c497f5-bcglg_kserve-ci-e2e-test(6f731566-8fe5-4ec1-8d1d-e8fa572f4efc) metadata: creationTimestamp: "2026-04-20T20:25:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:09Z" name: isvc-secondary-7f229f-predictor-6cb8c497f5-bcglg.18a82a6f13847493 namespace: kserve-ci-e2e-test resourceVersion: "19022" uid: 871ef6f9-d1bb-417a-9c4b-ca41ddbdabc3 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-7f229f-predictor-6cb8c497f5 namespace: kserve-ci-e2e-test resourceVersion: "18932" uid: e8a1035b-82ef-4a26-8f5b-d52efab9ffad kind: Event lastTimestamp: "2026-04-20T20:25:01Z" message: 'Created pod: isvc-secondary-7f229f-predictor-6cb8c497f5-bcglg' metadata: creationTimestamp: "2026-04-20T20:25:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:01Z" name: isvc-secondary-7f229f-predictor-6cb8c497f5.18a82a6d49bbfe07 namespace: kserve-ci-e2e-test resourceVersion: "18942" uid: 99cd7073-5502-4050-9f0d-5267915aa738 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-7f229f-predictor namespace: kserve-ci-e2e-test resourceVersion: "18930" uid: 4ca0da3c-8bf1-4d78-8825-3a96e3e00a9a kind: Event lastTimestamp: "2026-04-20T20:25:01Z" message: Scaled up replica set isvc-secondary-7f229f-predictor-6cb8c497f5 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:25:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:01Z" name: isvc-secondary-7f229f-predictor.18a82a6d46fb97ea namespace: kserve-ci-e2e-test resourceVersion: "18934" uid: c2d373c1-8f96-498a-87e0-1c724dd43833 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-20T20:25:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-7f229f namespace: kserve-ci-e2e-test resourceVersion: "18921" uid: 8552fdcd-227c-40e8-81a2-6e250a0cc20e kind: Event lastTimestamp: "2026-04-20T20:25:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:25:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:12Z" name: isvc-secondary-7f229f.18a82a6d46d5adca namespace: kserve-ci-e2e-test resourceVersion: "19048" uid: 727b4d07-1940-4f0a-b862-c18e040f1c6b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-7f229f namespace: kserve-ci-e2e-test resourceVersion: "18921" uid: 8552fdcd-227c-40e8-81a2-6e250a0cc20e kind: Event lastTimestamp: "2026-04-20T20:25:01Z" message: 'Failed to update status for InferenceService "isvc-secondary-7f229f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-7f229f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:25:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:01Z" name: isvc-secondary-7f229f.18a82a6d4726f230 namespace: kserve-ci-e2e-test resourceVersion: "18936" uid: 6bb8c38c-bd54-43ca-a92a-e2fc33d5b50d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-7f229f namespace: kserve-ci-e2e-test resourceVersion: "18921" uid: 8552fdcd-227c-40e8-81a2-6e250a0cc20e kind: Event lastTimestamp: "2026-04-20T20:25:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-7f229f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:25:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:01Z" name: isvc-secondary-7f229f.18a82a6d47273c46 namespace: kserve-ci-e2e-test resourceVersion: "18937" uid: a8d849ee-8fb9-408d-9401-4e659736a698 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:16:05.631481Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv namespace: kserve-ci-e2e-test resourceVersion: "14242" uid: b5b9121c-dafe-40ae-8da8-c95937dd3823 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv to ip-10-0-135-184.ec2.internal metadata: creationTimestamp: "2026-04-20T20:16:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:16:05Z" name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv.18a829f06f28b4ac namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: ffe53143-d87d-4627-9701-fefe4b857266 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5766b54df6-m4bt5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: b5b9121c-dafe-40ae-8da8-c95937dd3823 kind: Event lastTimestamp: "2026-04-20T20:16:06Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:16:06Z" name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv.18a829f088a5a4ce namespace: kserve-ci-e2e-test resourceVersion: "14290" uid: 967f4d0e-fb36-4a46-b7f3-ca04478fcc40 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: b5b9121c-dafe-40ae-8da8-c95937dd3823 kind: Event lastTimestamp: "2026-04-20T20:16:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" already present on machine metadata: creationTimestamp: "2026-04-20T20:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:06Z" name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv.18a829f08a2aed7c namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: 41659a03-23b8-4fb0-93ab-177d006c2321 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: b5b9121c-dafe-40ae-8da8-c95937dd3823 kind: Event lastTimestamp: "2026-04-20T20:16:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-20T20:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:06Z" name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv.18a829f08d4c969f namespace: kserve-ci-e2e-test resourceVersion: "14296" uid: c58b506e-0ebd-4221-a6fc-4288491ec64f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: b5b9121c-dafe-40ae-8da8-c95937dd3823 kind: Event lastTimestamp: "2026-04-20T20:16:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-20T20:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:06Z" name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv.18a829f08dd1d50f namespace: kserve-ci-e2e-test resourceVersion: "14297" uid: fb6ef5c1-ff7c-4f0a-9771-d86069844896 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: b5b9121c-dafe-40ae-8da8-c95937dd3823 kind: Event lastTimestamp: "2026-04-20T20:16:10Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:16:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:10Z" name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv.18a829f19a58d38a namespace: kserve-ci-e2e-test resourceVersion: "14353" uid: 8cbb1cf2-0219-47fb-b95e-1337cb0b2394 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: b5b9121c-dafe-40ae-8da8-c95937dd3823 kind: Event lastTimestamp: "2026-04-20T20:16:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:16:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:10Z" name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv.18a829f19dab22a2 namespace: kserve-ci-e2e-test resourceVersion: "14359" uid: 339469d4-9bd5-45ea-a52d-3f7090c738d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: b5b9121c-dafe-40ae-8da8-c95937dd3823 kind: Event lastTimestamp: "2026-04-20T20:16:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:16:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:10Z" name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv.18a829f19e28fc11 namespace: kserve-ci-e2e-test resourceVersion: "14360" uid: 8dabca27-c250-4b53-84c9-919eaf60b9db reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: b5b9121c-dafe-40ae-8da8-c95937dd3823 kind: Event lastTimestamp: "2026-04-20T20:16:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:16:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:10Z" name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv.18a829f19e37fd60 namespace: kserve-ci-e2e-test resourceVersion: "14361" uid: ef918e0f-4811-4347-a915-192e492bf69e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: b5b9121c-dafe-40ae-8da8-c95937dd3823 kind: Event lastTimestamp: "2026-04-20T20:16:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:16:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:10Z" name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv.18a829f1a1441b8c namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 30a3c333-4d89-4ac5-9f74-86d9f56fee00 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: b5b9121c-dafe-40ae-8da8-c95937dd3823 kind: Event lastTimestamp: "2026-04-20T20:16:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:16:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:10Z" name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv.18a829f1a1c0bef8 namespace: kserve-ci-e2e-test resourceVersion: "14365" uid: 2ae3005b-0b81-4fca-91b5-09ed6d389a7d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-20T20:16:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: b5b9121c-dafe-40ae-8da8-c95937dd3823 kind: Event lastTimestamp: "2026-04-20T20:17:58Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:16:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:58Z" name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv.18a829f2127810ba namespace: kserve-ci-e2e-test resourceVersion: "15384" uid: 01f9d41f-b50b-4b3f-88a7-df4220c4a9c8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: b5b9121c-dafe-40ae-8da8-c95937dd3823 kind: Event lastTimestamp: "2026-04-20T20:17:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:55Z" name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv.18a82a0a193cc3c7 namespace: kserve-ci-e2e-test resourceVersion: "15268" uid: 899a251a-74c5-4c11-a558-951a1d9aa1f0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: b5b9121c-dafe-40ae-8da8-c95937dd3823 kind: Event lastTimestamp: "2026-04-20T20:17:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:55Z" name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv.18a82a0a193de1f7 namespace: kserve-ci-e2e-test resourceVersion: "15271" uid: f98a4581-508a-4b2e-b7e9-8e63b7a04321 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: b5b9121c-dafe-40ae-8da8-c95937dd3823 kind: Event lastTimestamp: "2026-04-20T20:17:58Z" message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp 10.133.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:17:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:58Z" name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv.18a82a0ac0b9b4f8 namespace: kserve-ci-e2e-test resourceVersion: "15383" uid: 3b1bc889-1100-4632-89ca-df2fd4bd01f7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c namespace: kserve-ci-e2e-test resourceVersion: "14229" uid: ddb8d3ae-d550-4222-b229-9d05f3d1d9ed kind: Event lastTimestamp: "2026-04-20T20:16:05Z" message: 'Created pod: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c-m5xwv' metadata: creationTimestamp: "2026-04-20T20:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:05Z" name: isvc-sklearn-graph-raw-87d08-predictor-646b446c9c.18a829f06eb73f9d namespace: kserve-ci-e2e-test resourceVersion: "14244" uid: 552b831d-2762-4799-933d-eb1f1cabd24f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-87d08-predictor namespace: kserve-ci-e2e-test resourceVersion: "14226" uid: 45f1c2c4-dbd3-4a72-bdf9-0da5efe4069f kind: Event lastTimestamp: "2026-04-20T20:16:05Z" message: Scaled up replica set isvc-sklearn-graph-raw-87d08-predictor-646b446c9c from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:05Z" name: isvc-sklearn-graph-raw-87d08-predictor.18a829f06a6ef12b namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 4448343d-7524-45ba-9120-ee79de954be8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-87d08-predictor namespace: kserve-ci-e2e-test resourceVersion: "14228" uid: 5f971b59-2273-44b4-a248-5c52704fdb29 kind: Event lastTimestamp: "2026-04-20T20:16:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:16:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:20Z" name: isvc-sklearn-graph-raw-87d08-predictor.18a829f3e96a17b2 namespace: kserve-ci-e2e-test resourceVersion: "14443" uid: ce06ead4-fcfd-42b4-a435-8e2ccca99d1c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-87d08-predictor namespace: kserve-ci-e2e-test resourceVersion: "14228" uid: 5f971b59-2273-44b4-a248-5c52704fdb29 kind: Event lastTimestamp: "2026-04-20T20:16:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:16:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:20Z" name: isvc-sklearn-graph-raw-87d08-predictor.18a829f3e9d2ae86 namespace: kserve-ci-e2e-test resourceVersion: "14445" uid: 40738ced-a4e0-4efc-af6c-615d44ffa09b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:16:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-87d08-predictor namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 5f971b59-2273-44b4-a248-5c52704fdb29 kind: Event lastTimestamp: "2026-04-20T20:17:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:16:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:17:50Z" name: isvc-sklearn-graph-raw-87d08-predictor.18a829f76877d06d namespace: kserve-ci-e2e-test resourceVersion: "15172" uid: 6799feb5-254f-4766-8d3a-e07bf5fd3fbc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:16:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-87d08-predictor namespace: kserve-ci-e2e-test resourceVersion: "14444" uid: 5f971b59-2273-44b4-a248-5c52704fdb29 kind: Event lastTimestamp: "2026-04-20T20:17:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:16:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:17:50Z" name: isvc-sklearn-graph-raw-87d08-predictor.18a829f768d7e195 namespace: kserve-ci-e2e-test resourceVersion: "15173" uid: bd26102d-92d1-4b7c-bf3c-c3ffe6ad703c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-20T20:16:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-87d08 namespace: kserve-ci-e2e-test resourceVersion: "14197" uid: 2e87ff8c-886f-4b9a-943b-70491aeb1eab kind: Event lastTimestamp: "2026-04-20T20:17:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:17:18Z" name: isvc-sklearn-graph-raw-87d08.18a829f06a4eed8f namespace: kserve-ci-e2e-test resourceVersion: "14891" uid: 4311f7fc-2bee-4c71-b606-fcaed987d0c6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-87d08 namespace: kserve-ci-e2e-test resourceVersion: "14197" uid: 2e87ff8c-886f-4b9a-943b-70491aeb1eab kind: Event lastTimestamp: "2026-04-20T20:16:05Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-87d08": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-87d08": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:16:05Z" name: isvc-sklearn-graph-raw-87d08.18a829f06ab780b0 namespace: kserve-ci-e2e-test resourceVersion: "14233" uid: 8e8c6017-6284-4cba-9471-8d0ccc521443 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-87d08 namespace: kserve-ci-e2e-test resourceVersion: "14889" uid: 2e87ff8c-886f-4b9a-943b-70491aeb1eab kind: Event lastTimestamp: "2026-04-20T20:17:18Z" message: InferenceService [isvc-sklearn-graph-raw-87d08] is Ready metadata: creationTimestamp: "2026-04-20T20:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:17:18Z" name: isvc-sklearn-graph-raw-87d08.18a82a0175b9b7f0 namespace: kserve-ci-e2e-test resourceVersion: "14890" uid: 610a9f79-592a-4555-8ab1-f67e14bc0ef5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:17:55.923165Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: f367b892-3117-4f93-b513-6a8a323a86e4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs to ip-10-0-135-184.ec2.internal metadata: creationTimestamp: "2026-04-20T20:17:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:17:55Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs.18a82a0a1d0e4117 namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 0b4f5fb3-cbcc-48f0-bf31-8429d73fd259 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5766b54df6-m4bt5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: f367b892-3117-4f93-b513-6a8a323a86e4 kind: Event lastTimestamp: "2026-04-20T20:17:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-hpa-3bab1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:56Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs.18a82a0a2829e247 namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: 4d2769a5-6028-4572-84f8-e483ce8fdbe2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: f367b892-3117-4f93-b513-6a8a323a86e4 kind: Event lastTimestamp: "2026-04-20T20:17:56Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:17:56Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs.18a82a0a5a04c96e namespace: kserve-ci-e2e-test resourceVersion: "15349" uid: 737c9d79-d277-4c12-958a-f4feaf1da2ca reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: f367b892-3117-4f93-b513-6a8a323a86e4 kind: Event lastTimestamp: "2026-04-20T20:17:56Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" already present on machine metadata: creationTimestamp: "2026-04-20T20:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:56Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs.18a82a0a5b631dbc namespace: kserve-ci-e2e-test resourceVersion: "15351" uid: fc01bc6e-dcfd-4314-88ca-31589a2fbc8c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: f367b892-3117-4f93-b513-6a8a323a86e4 kind: Event lastTimestamp: "2026-04-20T20:17:57Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-20T20:17:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:57Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs.18a82a0a5fa84ce6 namespace: kserve-ci-e2e-test resourceVersion: "15353" uid: 9e6adbde-a836-418a-8f88-624b2248ddbf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: f367b892-3117-4f93-b513-6a8a323a86e4 kind: Event lastTimestamp: "2026-04-20T20:17:57Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-20T20:17:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:57Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs.18a82a0a602c0ed2 namespace: kserve-ci-e2e-test resourceVersion: "15354" uid: fc0890a4-c691-4d07-b11a-978915f58271 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:18:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: f367b892-3117-4f93-b513-6a8a323a86e4 kind: Event lastTimestamp: "2026-04-20T20:18:01Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:18:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:18:01Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs.18a82a0b4f279495 namespace: kserve-ci-e2e-test resourceVersion: "15411" uid: 03c4fd4a-0c48-432e-bd37-492074719095 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:18:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: f367b892-3117-4f93-b513-6a8a323a86e4 kind: Event lastTimestamp: "2026-04-20T20:18:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:18:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:18:01Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs.18a82a0b54ae482c namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 7280231b-cbf4-4224-8db0-de48f24712b2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:18:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: f367b892-3117-4f93-b513-6a8a323a86e4 kind: Event lastTimestamp: "2026-04-20T20:18:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:18:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:18:01Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs.18a82a0b5532988f namespace: kserve-ci-e2e-test resourceVersion: "15422" uid: b6d8d7f3-fcaa-4c2c-a24e-5d40f9f1b625 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:18:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: f367b892-3117-4f93-b513-6a8a323a86e4 kind: Event lastTimestamp: "2026-04-20T20:18:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:18:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:18:01Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs.18a82a0b554079dc namespace: kserve-ci-e2e-test resourceVersion: "15424" uid: 9bca3f73-938a-4cab-a242-60762f35b1e7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:18:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: f367b892-3117-4f93-b513-6a8a323a86e4 kind: Event lastTimestamp: "2026-04-20T20:18:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:18:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:18:01Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs.18a82a0b58ba2ee9 namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 6f819fc0-0c50-4af3-aee2-f3eb3f0d4e2e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:18:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: f367b892-3117-4f93-b513-6a8a323a86e4 kind: Event lastTimestamp: "2026-04-20T20:18:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:18:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:18:01Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs.18a82a0b59390bf5 namespace: kserve-ci-e2e-test resourceVersion: "15431" uid: d24ce041-5e2c-4874-ba85-7652b05730c0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-20T20:18:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: f367b892-3117-4f93-b513-6a8a323a86e4 kind: Event lastTimestamp: "2026-04-20T20:19:38Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:38Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs.18a82a0b8b1ebb5f namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: baac21c7-0b1c-483c-b26b-9966282e502d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: f367b892-3117-4f93-b513-6a8a323a86e4 kind: Event lastTimestamp: "2026-04-20T20:19:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:36Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs.18a82a217381c138 namespace: kserve-ci-e2e-test resourceVersion: "16242" uid: fcf92511-12bf-4c32-a437-fb92361e729a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: f367b892-3117-4f93-b513-6a8a323a86e4 kind: Event lastTimestamp: "2026-04-20T20:19:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:36Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs.18a82a2173825d2f namespace: kserve-ci-e2e-test resourceVersion: "16243" uid: eee5d365-48e0-4104-a508-c30b997c3374 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: f367b892-3117-4f93-b513-6a8a323a86e4 kind: Event lastTimestamp: "2026-04-20T20:19:38Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:19:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:38Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs.18a82a21e57a980d namespace: kserve-ci-e2e-test resourceVersion: "16301" uid: b03aa4b7-b485-4ca5-b7b1-8e0c656b7616 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f namespace: kserve-ci-e2e-test resourceVersion: "15256" uid: afe6f910-5f40-4581-8431-3e2dbd68e726 kind: Event lastTimestamp: "2026-04-20T20:17:55Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f-lg7rs' metadata: creationTimestamp: "2026-04-20T20:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:17:55Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f.18a82a0a1c72c8f4 namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: 05e3c78c-6c1f-4924-ad0f-e04f74ec8ec2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-3bab1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15248" uid: 82759e33-4d17-4276-9f53-c5971d2b7025 kind: Event lastTimestamp: "2026-04-20T20:17:55Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-3bab1-predictor-69b497f96f from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:17:55Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor.18a82a0a18614080 namespace: kserve-ci-e2e-test resourceVersion: "15260" uid: 984ec02c-55ec-45e1-9c16-80767d617a9e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:18:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3bab1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: d6f985a9-a726-4a7d-a1af-2c222e54b834 kind: Event lastTimestamp: "2026-04-20T20:18:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:18:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:18:25Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor.18a82a0d9743198f namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: 165ae79d-fa4f-4692-a0da-6211cc2da8db reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:18:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3bab1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: d6f985a9-a726-4a7d-a1af-2c222e54b834 kind: Event lastTimestamp: "2026-04-20T20:18:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:18:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:18:25Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor.18a82a0d97a9fb86 namespace: kserve-ci-e2e-test resourceVersion: "15620" uid: e36bfbae-001d-4d8a-ac44-99a6286320ff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:18:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3bab1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15512" uid: d6f985a9-a726-4a7d-a1af-2c222e54b834 kind: Event lastTimestamp: "2026-04-20T20:19:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:18:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:19:25Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor.18a82a149594eccf namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: 93182c52-379b-4cc9-af03-b25615094ca6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:18:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3bab1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15512" uid: d6f985a9-a726-4a7d-a1af-2c222e54b834 kind: Event lastTimestamp: "2026-04-20T20:19:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:18:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:19:25Z" name: isvc-sklearn-graph-raw-hpa-3bab1-predictor.18a82a1495f83e23 namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: 3fa5d513-1e34-4bfe-bacd-e7cb8adb0f0e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-20T20:17:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-3bab1 namespace: kserve-ci-e2e-test resourceVersion: "15236" uid: 012820c8-bd81-48b3-9178-026650fe8b83 kind: Event lastTimestamp: "2026-04-20T20:19:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:19:08Z" name: isvc-sklearn-graph-raw-hpa-3bab1.18a82a0a183a7647 namespace: kserve-ci-e2e-test resourceVersion: "15949" uid: 00ef559a-2b45-41b5-a69f-d71407e02519 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-3bab1 namespace: kserve-ci-e2e-test resourceVersion: "15236" uid: 012820c8-bd81-48b3-9178-026650fe8b83 kind: Event lastTimestamp: "2026-04-20T20:17:55Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-3bab1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-3bab1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:17:55Z" name: isvc-sklearn-graph-raw-hpa-3bab1.18a82a0a188d6a67 namespace: kserve-ci-e2e-test resourceVersion: "15265" uid: d5a7e3d8-9483-497c-9331-91cfd86b8291 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-3bab1 namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 012820c8-bd81-48b3-9178-026650fe8b83 kind: Event lastTimestamp: "2026-04-20T20:19:08Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-3bab1] is Ready metadata: creationTimestamp: "2026-04-20T20:19:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:19:08Z" name: isvc-sklearn-graph-raw-hpa-3bab1.18a82a1aee4adc76 namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: fba86c49-aac1-4170-8e8c-76610199b2ee reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:21:21.481379Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6 namespace: kserve-ci-e2e-test resourceVersion: "17188" uid: 91d4ea93-1078-4593-a434-000073a3242a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6 to ip-10-0-135-184.ec2.internal metadata: creationTimestamp: "2026-04-20T20:21:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:21:21Z" name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6.18a82a39f947b360 namespace: kserve-ci-e2e-test resourceVersion: "17195" uid: 50d33b09-be1f-4d1e-8850-3cc873dca956 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5766b54df6-m4bt5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:21:21Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6 namespace: kserve-ci-e2e-test resourceVersion: "17198" uid: 91d4ea93-1078-4593-a434-000073a3242a kind: Event lastTimestamp: "2026-04-20T20:21:21Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:21:21Z" name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6.18a82a3a12b36fca namespace: kserve-ci-e2e-test resourceVersion: "17222" uid: 232176db-37bb-4d39-87aa-36b9e9016691 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:21:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: 91d4ea93-1078-4593-a434-000073a3242a kind: Event lastTimestamp: "2026-04-20T20:21:21Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" already present on machine metadata: creationTimestamp: "2026-04-20T20:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:21:21Z" name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6.18a82a3a14113fae namespace: kserve-ci-e2e-test resourceVersion: "17224" uid: 9f3bc1b8-143f-4f1f-ac51-85b0ce596c20 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:21:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: 91d4ea93-1078-4593-a434-000073a3242a kind: Event lastTimestamp: "2026-04-20T20:21:21Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-20T20:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:21:21Z" name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6.18a82a3a171d5822 namespace: kserve-ci-e2e-test resourceVersion: "17225" uid: d3088667-7e70-4b28-a07a-2a185c02b9a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:21:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: 91d4ea93-1078-4593-a434-000073a3242a kind: Event lastTimestamp: "2026-04-20T20:21:21Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-20T20:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:21:21Z" name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6.18a82a3a179671b0 namespace: kserve-ci-e2e-test resourceVersion: "17226" uid: 734c753a-c70f-4115-9cac-d2d187d5e88c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:21:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: 91d4ea93-1078-4593-a434-000073a3242a kind: Event lastTimestamp: "2026-04-20T20:21:25Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:21:25Z" name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6.18a82a3af8d13c44 namespace: kserve-ci-e2e-test resourceVersion: "17263" uid: caca65ac-d174-442f-8b9e-fba6686d7ce8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:21:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: 91d4ea93-1078-4593-a434-000073a3242a kind: Event lastTimestamp: "2026-04-20T20:21:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:21:25Z" name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6.18a82a3afbcf0881 namespace: kserve-ci-e2e-test resourceVersion: "17267" uid: a4cc6cdb-ded4-4360-a343-bbd1334baf33 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:21:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: 91d4ea93-1078-4593-a434-000073a3242a kind: Event lastTimestamp: "2026-04-20T20:21:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:21:25Z" name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6.18a82a3afc426018 namespace: kserve-ci-e2e-test resourceVersion: "17268" uid: 9a4096d2-5a1b-4ee4-96d5-edf3bf293973 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:21:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: 91d4ea93-1078-4593-a434-000073a3242a kind: Event lastTimestamp: "2026-04-20T20:21:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:21:25Z" name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6.18a82a3afc4fc49f namespace: kserve-ci-e2e-test resourceVersion: "17269" uid: 3ea41ab1-1c06-4ca6-a16a-75bce616fbc4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:21:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: 91d4ea93-1078-4593-a434-000073a3242a kind: Event lastTimestamp: "2026-04-20T20:21:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:21:25Z" name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6.18a82a3aff9a15a5 namespace: kserve-ci-e2e-test resourceVersion: "17271" uid: 4e3d0c43-a722-4dd0-b6f9-f89a91f04e10 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:21:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: 91d4ea93-1078-4593-a434-000073a3242a kind: Event lastTimestamp: "2026-04-20T20:21:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:21:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:21:25Z" name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6.18a82a3b0011eaa9 namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: ac365fab-c22b-4766-aa67-bc267f7f6e7e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 16 eventTime: null firstTimestamp: "2026-04-20T20:21:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: 91d4ea93-1078-4593-a434-000073a3242a kind: Event lastTimestamp: "2026-04-20T20:23:48Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:23:48Z" name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6.18a82a3b34e74f31 namespace: kserve-ci-e2e-test resourceVersion: "18382" uid: e425807c-2b96-44c5-87aa-472518128b2d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:23:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: 91d4ea93-1078-4593-a434-000073a3242a kind: Event lastTimestamp: "2026-04-20T20:23:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:23:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:23:41Z" name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6.18a82a5a9d1b31b4 namespace: kserve-ci-e2e-test resourceVersion: "18283" uid: 47546b05-3781-4c12-a5cf-cce0520b836a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:23:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: 91d4ea93-1078-4593-a434-000073a3242a kind: Event lastTimestamp: "2026-04-20T20:23:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:23:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:23:41Z" name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6.18a82a5a9d1bbd54 namespace: kserve-ci-e2e-test resourceVersion: "18287" uid: 2e3925ab-dee6-4b61-8ea4-1418d7357bd3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:23:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6 namespace: kserve-ci-e2e-test resourceVersion: "17192" uid: 91d4ea93-1078-4593-a434-000073a3242a kind: Event lastTimestamp: "2026-04-20T20:23:47Z" message: 'Readiness probe failed: Get "https://10.133.0.39:8643/healthz": dial tcp 10.133.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:23:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:23:47Z" name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6.18a82a5adf4a8dce namespace: kserve-ci-e2e-test resourceVersion: "18374" uid: 6beb0586-9106-4dcc-aeb1-9ae3ff9690dc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:21:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67 namespace: kserve-ci-e2e-test resourceVersion: "17172" uid: c9c3fdfd-6c3b-4a1c-b1f4-1df3188e6887 kind: Event lastTimestamp: "2026-04-20T20:21:21Z" message: 'Created pod: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67-rvhq6' metadata: creationTimestamp: "2026-04-20T20:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:21:21Z" name: isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67.18a82a39f8c0acdb namespace: kserve-ci-e2e-test resourceVersion: "17191" uid: 5ec23e29-60c1-4d1f-a8bd-d7ad67579aba reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:21:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-30526-predictor namespace: kserve-ci-e2e-test resourceVersion: "17164" uid: 3a4f36a2-3d77-40cc-80bc-5d6f18e0d30e kind: Event lastTimestamp: "2026-04-20T20:21:21Z" message: Scaled up replica set isvc-sklearn-scale-raw-30526-predictor-66fb4d7b67 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:21:21Z" name: isvc-sklearn-scale-raw-30526-predictor.18a82a39f6018bab namespace: kserve-ci-e2e-test resourceVersion: "17175" uid: 6d114304-9efb-46e2-8c22-797093d99065 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:21:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-30526-predictor namespace: kserve-ci-e2e-test resourceVersion: "17168" uid: 68c394c8-2a79-428e-b863-89aa6ef09376 kind: Event lastTimestamp: "2026-04-20T20:21:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:21:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:21:51Z" name: isvc-sklearn-scale-raw-30526-predictor.18a82a3d7452de94 namespace: kserve-ci-e2e-test resourceVersion: "17470" uid: c4184c94-4fc9-4d18-9547-5035b1c7908b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:21:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-30526-predictor namespace: kserve-ci-e2e-test resourceVersion: "17168" uid: 68c394c8-2a79-428e-b863-89aa6ef09376 kind: Event lastTimestamp: "2026-04-20T20:21:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:21:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:21:51Z" name: isvc-sklearn-scale-raw-30526-predictor.18a82a3d74b37893 namespace: kserve-ci-e2e-test resourceVersion: "17471" uid: 9d09916e-2d59-4769-adf2-f8d6e03b9aab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-20T20:22:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-30526-predictor namespace: kserve-ci-e2e-test resourceVersion: "17363" uid: 68c394c8-2a79-428e-b863-89aa6ef09376 kind: Event lastTimestamp: "2026-04-20T20:23:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:23:36Z" name: isvc-sklearn-scale-raw-30526-predictor.18a82a4473830ee8 namespace: kserve-ci-e2e-test resourceVersion: "18204" uid: 4da8c770-10b8-4ec0-b3f9-36e0a4740d4d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-20T20:22:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-30526-predictor namespace: kserve-ci-e2e-test resourceVersion: "17363" uid: 68c394c8-2a79-428e-b863-89aa6ef09376 kind: Event lastTimestamp: "2026-04-20T20:23:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:23:36Z" name: isvc-sklearn-scale-raw-30526-predictor.18a82a4473e61acb namespace: kserve-ci-e2e-test resourceVersion: "18205" uid: 3138ad9a-251f-462e-8db6-ecfdbd215717 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-20T20:21:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-30526 namespace: kserve-ci-e2e-test resourceVersion: "17148" uid: 6267760e-9d5b-40f8-91c2-7f6252beb3cb kind: Event lastTimestamp: "2026-04-20T20:23:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:23:38Z" name: isvc-sklearn-scale-raw-30526.18a82a39f5b22333 namespace: kserve-ci-e2e-test resourceVersion: "18230" uid: aa6e0531-6ba3-4343-9442-463aa7a121e1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:21:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-30526 namespace: kserve-ci-e2e-test resourceVersion: "17148" uid: 6267760e-9d5b-40f8-91c2-7f6252beb3cb kind: Event lastTimestamp: "2026-04-20T20:21:21Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-30526": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-30526": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:21:21Z" name: isvc-sklearn-scale-raw-30526.18a82a39f64afcc1 namespace: kserve-ci-e2e-test resourceVersion: "17179" uid: a4c8e218-8391-4960-b891-f47fb379b075 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:23:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-30526 namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: 6267760e-9d5b-40f8-91c2-7f6252beb3cb kind: Event lastTimestamp: "2026-04-20T20:23:38Z" message: InferenceService [isvc-sklearn-scale-raw-30526] is Ready metadata: creationTimestamp: "2026-04-20T20:23:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:23:38Z" name: isvc-sklearn-scale-raw-30526.18a82a59e72a1a4e namespace: kserve-ci-e2e-test resourceVersion: "18229" uid: da644c68-35ed-4262-bb0f-5886624dbf35 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:16:05.707973Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22 namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 0dec70b8-66c8-428b-a746-6c4139f337ab kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22 to ip-10-0-135-184.ec2.internal metadata: creationTimestamp: "2026-04-20T20:16:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:16:05Z" name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22.18a829f073b7e1d1 namespace: kserve-ci-e2e-test resourceVersion: "14277" uid: 53841eca-194e-4649-9900-ee9b3c0511fd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5766b54df6-m4bt5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: 0dec70b8-66c8-428b-a746-6c4139f337ab kind: Event lastTimestamp: "2026-04-20T20:16:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-87d08-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:05Z" name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22.18a829f07d074bca namespace: kserve-ci-e2e-test resourceVersion: "14285" uid: fa13c0a2-328a-4d0b-83b9-d6e991763c0a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22 namespace: kserve-ci-e2e-test resourceVersion: "14283" uid: 0dec70b8-66c8-428b-a746-6c4139f337ab kind: Event lastTimestamp: "2026-04-20T20:16:06Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:16:06Z" name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22.18a829f0b196f6f4 namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: 9880ef42-fca4-4682-a92f-7f86eceab1ca reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: 0dec70b8-66c8-428b-a746-6c4139f337ab kind: Event lastTimestamp: "2026-04-20T20:16:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" already present on machine metadata: creationTimestamp: "2026-04-20T20:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:06Z" name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22.18a829f0b2de5d43 namespace: kserve-ci-e2e-test resourceVersion: "14307" uid: 9c4386ae-1629-471f-bee9-f515d886ce53 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: 0dec70b8-66c8-428b-a746-6c4139f337ab kind: Event lastTimestamp: "2026-04-20T20:16:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-20T20:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:06Z" name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22.18a829f0b620a1b2 namespace: kserve-ci-e2e-test resourceVersion: "14308" uid: db1b0418-32fc-4afc-b8eb-1359d0a96925 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: 0dec70b8-66c8-428b-a746-6c4139f337ab kind: Event lastTimestamp: "2026-04-20T20:16:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-20T20:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:06Z" name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22.18a829f0b69975c1 namespace: kserve-ci-e2e-test resourceVersion: "14309" uid: f2b6d509-dec5-41e4-a0ec-5e854cf2560f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: 0dec70b8-66c8-428b-a746-6c4139f337ab kind: Event lastTimestamp: "2026-04-20T20:16:10Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-20T20:16:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:10Z" name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22.18a829f19a72e7a7 namespace: kserve-ci-e2e-test resourceVersion: "14354" uid: 34a0e24f-c21e-4fda-8683-cc8929dd86ff reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: 0dec70b8-66c8-428b-a746-6c4139f337ab kind: Event lastTimestamp: "2026-04-20T20:16:34Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 23.62s (23.62s including waiting). Image size: 1306414331 bytes.' metadata: creationTimestamp: "2026-04-20T20:16:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:34Z" name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22.18a829f71a5d40e6 namespace: kserve-ci-e2e-test resourceVersion: "14548" uid: ac59a789-8858-4da6-8410-14f0e6f1fee1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: 0dec70b8-66c8-428b-a746-6c4139f337ab kind: Event lastTimestamp: "2026-04-20T20:16:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:16:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:34Z" name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22.18a829f71dab15ab namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: b1383d16-3548-4941-ad83-aafdd17fbd86 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: 0dec70b8-66c8-428b-a746-6c4139f337ab kind: Event lastTimestamp: "2026-04-20T20:16:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:16:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:34Z" name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22.18a829f71e24651c namespace: kserve-ci-e2e-test resourceVersion: "14550" uid: 7f6b7c31-f9f3-4e61-9c8d-2c9a46596bd5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: 0dec70b8-66c8-428b-a746-6c4139f337ab kind: Event lastTimestamp: "2026-04-20T20:16:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:16:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:34Z" name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22.18a829f71e3326f5 namespace: kserve-ci-e2e-test resourceVersion: "14551" uid: 04000147-2ebd-4930-be61-fdd13ca740cd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: 0dec70b8-66c8-428b-a746-6c4139f337ab kind: Event lastTimestamp: "2026-04-20T20:16:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:16:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:34Z" name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22.18a829f72275dc01 namespace: kserve-ci-e2e-test resourceVersion: "14553" uid: 237833c4-efb6-4938-b329-a45705a0aa66 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: 0dec70b8-66c8-428b-a746-6c4139f337ab kind: Event lastTimestamp: "2026-04-20T20:16:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:16:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:16:34Z" name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22.18a829f722f3bfe4 namespace: kserve-ci-e2e-test resourceVersion: "14554" uid: e373f373-c31f-46ed-861a-df37c48c7b7e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-20T20:16:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: 0dec70b8-66c8-428b-a746-6c4139f337ab kind: Event lastTimestamp: "2026-04-20T20:17:31Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:16:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:31Z" name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22.18a829f772c5c55a namespace: kserve-ci-e2e-test resourceVersion: "14989" uid: 009cf23e-303e-4a78-a2bd-7808bac96641 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: 0dec70b8-66c8-428b-a746-6c4139f337ab kind: Event lastTimestamp: "2026-04-20T20:17:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:56Z" name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22.18a82a0a222380a4 namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: 7f95f51a-4bc6-4fae-9bd9-e8bc1bc63f1f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: 0dec70b8-66c8-428b-a746-6c4139f337ab kind: Event lastTimestamp: "2026-04-20T20:17:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:56Z" name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22.18a82a0a22244d79 namespace: kserve-ci-e2e-test resourceVersion: "15319" uid: 3e96b210-8558-4417-8638-1d1c4024c0f9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22 namespace: kserve-ci-e2e-test resourceVersion: "14273" uid: 0dec70b8-66c8-428b-a746-6c4139f337ab kind: Event lastTimestamp: "2026-04-20T20:17:56Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:56Z" name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22.18a82a0a4ef02284 namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: b430128c-5bb3-4b56-b51c-b5f0b4012a85 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457 namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 0581848c-dbd4-45f2-b036-d55a0eac1f66 kind: Event lastTimestamp: "2026-04-20T20:16:05Z" message: 'Created pod: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457-kpb22' metadata: creationTimestamp: "2026-04-20T20:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:05Z" name: isvc-xgboost-graph-raw-87d08-predictor-84bdc99457.18a829f073549317 namespace: kserve-ci-e2e-test resourceVersion: "14272" uid: f6c68ee9-329c-4f37-b36a-6fab0170fa60 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-87d08-predictor namespace: kserve-ci-e2e-test resourceVersion: "14254" uid: 6e5fc32e-8316-4054-a9e5-cbfd5ef349de kind: Event lastTimestamp: "2026-04-20T20:16:05Z" message: Scaled up replica set isvc-xgboost-graph-raw-87d08-predictor-84bdc99457 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:16:05Z" name: isvc-xgboost-graph-raw-87d08-predictor.18a829f07083f367 namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: f8b23994-6de3-4c20-a26b-3fab3234893b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:16:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-87d08-predictor namespace: kserve-ci-e2e-test resourceVersion: "14258" uid: 86cb71cf-016f-4e59-8e7b-5d1829b4a56b kind: Event lastTimestamp: "2026-04-20T20:17:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:16:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:17:05Z" name: isvc-xgboost-graph-raw-87d08-predictor.18a829f3ef5797e7 namespace: kserve-ci-e2e-test resourceVersion: "14789" uid: 7e7809cc-c376-4157-a321-45a1b4eccf66 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-20T20:16:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-87d08-predictor namespace: kserve-ci-e2e-test resourceVersion: "14258" uid: 86cb71cf-016f-4e59-8e7b-5d1829b4a56b kind: Event lastTimestamp: "2026-04-20T20:17:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:16:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:17:05Z" name: isvc-xgboost-graph-raw-87d08-predictor.18a829f3efb66c0c namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: 02783e61-a3a2-4212-8fe4-6aa158de7f0a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:16:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-87d08-predictor namespace: kserve-ci-e2e-test resourceVersion: "14447" uid: 86cb71cf-016f-4e59-8e7b-5d1829b4a56b kind: Event lastTimestamp: "2026-04-20T20:17:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:16:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:17:50Z" name: isvc-xgboost-graph-raw-87d08-predictor.18a829faed1643c7 namespace: kserve-ci-e2e-test resourceVersion: "15174" uid: 4979d95d-95e5-4fc5-aa55-0c212ec455cf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:16:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-87d08-predictor namespace: kserve-ci-e2e-test resourceVersion: "14447" uid: 86cb71cf-016f-4e59-8e7b-5d1829b4a56b kind: Event lastTimestamp: "2026-04-20T20:17:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:16:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:17:50Z" name: isvc-xgboost-graph-raw-87d08-predictor.18a829faed837a39 namespace: kserve-ci-e2e-test resourceVersion: "15175" uid: ce6de48d-4f6c-47f5-9a11-7a3d43ca08ee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-20T20:16:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-87d08 namespace: kserve-ci-e2e-test resourceVersion: "14238" uid: 894df369-b816-49c4-b4db-d33779ab9a94 kind: Event lastTimestamp: "2026-04-20T20:17:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:17:41Z" name: isvc-xgboost-graph-raw-87d08.18a829f070b8b1b8 namespace: kserve-ci-e2e-test resourceVersion: "15071" uid: ec79e9e0-aff6-4258-bf75-cd8ca4dbd4c3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:16:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-87d08 namespace: kserve-ci-e2e-test resourceVersion: "14238" uid: 894df369-b816-49c4-b4db-d33779ab9a94 kind: Event lastTimestamp: "2026-04-20T20:16:05Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-87d08": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-87d08": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:16:05Z" name: isvc-xgboost-graph-raw-87d08.18a829f0712b9dc2 namespace: kserve-ci-e2e-test resourceVersion: "14266" uid: e4607ad5-6750-4b55-bbc2-161675c19fee reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-87d08 namespace: kserve-ci-e2e-test resourceVersion: "15068" uid: 894df369-b816-49c4-b4db-d33779ab9a94 kind: Event lastTimestamp: "2026-04-20T20:17:41Z" message: InferenceService [isvc-xgboost-graph-raw-87d08] is Ready metadata: creationTimestamp: "2026-04-20T20:17:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:17:41Z" name: isvc-xgboost-graph-raw-87d08.18a82a06d58e72ae namespace: kserve-ci-e2e-test resourceVersion: "15069" uid: 275badda-c409-40c0-ae03-724ee55e1b70 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:17:56.038523Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6 namespace: kserve-ci-e2e-test resourceVersion: "15321" uid: 86334311-672d-4ba3-b0d8-59aa9263198f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6 to ip-10-0-135-184.ec2.internal metadata: creationTimestamp: "2026-04-20T20:17:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:17:56Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6.18a82a0a23ee86f4 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 208db003-f695-48e2-94da-9cb15b617da6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5766b54df6-m4bt5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 86334311-672d-4ba3-b0d8-59aa9263198f kind: Event lastTimestamp: "2026-04-20T20:17:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-hpa-3bab1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:56Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6.18a82a0a2e33e9b7 namespace: kserve-ci-e2e-test resourceVersion: "15338" uid: b2294351-86d1-440f-8672-26001ae881e3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6 namespace: kserve-ci-e2e-test resourceVersion: "15328" uid: 86334311-672d-4ba3-b0d8-59aa9263198f kind: Event lastTimestamp: "2026-04-20T20:17:57Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:17:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:17:57Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6.18a82a0a617851da namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: aa021e39-ab5a-47ee-96be-5df8131449a4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 86334311-672d-4ba3-b0d8-59aa9263198f kind: Event lastTimestamp: "2026-04-20T20:17:57Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" already present on machine metadata: creationTimestamp: "2026-04-20T20:17:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:57Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6.18a82a0a6321935a namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: d7656840-81f9-4e8d-be5d-c6333623c147 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 86334311-672d-4ba3-b0d8-59aa9263198f kind: Event lastTimestamp: "2026-04-20T20:17:57Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-20T20:17:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:57Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6.18a82a0a66484f78 namespace: kserve-ci-e2e-test resourceVersion: "15358" uid: 157c0f6e-2e0f-4663-afdb-04f34c6f0b91 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 86334311-672d-4ba3-b0d8-59aa9263198f kind: Event lastTimestamp: "2026-04-20T20:17:57Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-20T20:17:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:57Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6.18a82a0a66bb3c8c namespace: kserve-ci-e2e-test resourceVersion: "15359" uid: 178026ca-d255-4e32-baa2-676e14f2baba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:18:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 86334311-672d-4ba3-b0d8-59aa9263198f kind: Event lastTimestamp: "2026-04-20T20:18:01Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-04-20T20:18:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:18:01Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6.18a82a0b4eecf76d namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 4f5b639a-0876-4e5e-bb8f-e74ce7244376 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:18:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 86334311-672d-4ba3-b0d8-59aa9263198f kind: Event lastTimestamp: "2026-04-20T20:18:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:18:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:18:01Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6.18a82a0b54b85fb6 namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 158199c7-b64f-45c5-9d0e-bc5f6ed3b0f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:18:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 86334311-672d-4ba3-b0d8-59aa9263198f kind: Event lastTimestamp: "2026-04-20T20:18:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:18:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:18:01Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6.18a82a0b553f6773 namespace: kserve-ci-e2e-test resourceVersion: "15423" uid: aafd3169-3859-4fba-a703-3f0dad9f9dc3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:18:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 86334311-672d-4ba3-b0d8-59aa9263198f kind: Event lastTimestamp: "2026-04-20T20:18:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:18:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:18:01Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6.18a82a0b554b33d1 namespace: kserve-ci-e2e-test resourceVersion: "15425" uid: 8a637614-0dda-40e4-9092-d1ead3046702 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:18:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 86334311-672d-4ba3-b0d8-59aa9263198f kind: Event lastTimestamp: "2026-04-20T20:18:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:18:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:18:01Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6.18a82a0b58adb843 namespace: kserve-ci-e2e-test resourceVersion: "15428" uid: 87aadea4-4adc-4120-85f0-c697d58f8138 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:18:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 86334311-672d-4ba3-b0d8-59aa9263198f kind: Event lastTimestamp: "2026-04-20T20:18:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:18:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:18:01Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6.18a82a0b592bd625 namespace: kserve-ci-e2e-test resourceVersion: "15430" uid: 86a88192-6276-4a67-97b5-20edeacb19aa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-20T20:18:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 86334311-672d-4ba3-b0d8-59aa9263198f kind: Event lastTimestamp: "2026-04-20T20:19:38Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:38Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6.18a82a0b8b34cc96 namespace: kserve-ci-e2e-test resourceVersion: "16302" uid: 7fc98489-6c11-4929-a67d-d2c20aae0fd4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 86334311-672d-4ba3-b0d8-59aa9263198f kind: Event lastTimestamp: "2026-04-20T20:19:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:36Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6.18a82a217b072ec3 namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: 3a3c5ac1-678a-4ec5-8b55-921ef46da7bf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 86334311-672d-4ba3-b0d8-59aa9263198f kind: Event lastTimestamp: "2026-04-20T20:19:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:36Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6.18a82a217b07ae6c namespace: kserve-ci-e2e-test resourceVersion: "16274" uid: 92540cce-e1bc-4eaf-85b4-6d398f5aa4f1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 86334311-672d-4ba3-b0d8-59aa9263198f kind: Event lastTimestamp: "2026-04-20T20:19:38Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:19:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:38Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6.18a82a21e57973c3 namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 8cdda030-ce26-4ed3-a1d1-47ccdc927b1b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9 namespace: kserve-ci-e2e-test resourceVersion: "15290" uid: 854e2ea5-849e-435d-ac63-ca0512e319d7 kind: Event lastTimestamp: "2026-04-20T20:17:56Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9-6kbd6' metadata: creationTimestamp: "2026-04-20T20:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:17:56Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9.18a82a0a2387dd72 namespace: kserve-ci-e2e-test resourceVersion: "15322" uid: 6ec8e870-d4ad-4a66-b46d-8fd50ab67a3c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-3bab1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15287" uid: 997da0cd-20b1-484f-89ef-c14d66dac014 kind: Event lastTimestamp: "2026-04-20T20:17:55Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-3bab1-predictor-858479d9d9 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:17:55Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor.18a82a0a1dac47e9 namespace: kserve-ci-e2e-test resourceVersion: "15294" uid: f66b178c-9244-404e-b903-a1cba6839af7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:18:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3bab1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15289" uid: 534a7c41-daa0-4f6d-ba73-6b808fd5b84b kind: Event lastTimestamp: "2026-04-20T20:18:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:18:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:18:40Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor.18a82a0d9c224dde namespace: kserve-ci-e2e-test resourceVersion: "15740" uid: 671d61f3-3820-4fb8-a338-fbb2355b40db reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:18:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3bab1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15289" uid: 534a7c41-daa0-4f6d-ba73-6b808fd5b84b kind: Event lastTimestamp: "2026-04-20T20:18:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:18:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:18:40Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor.18a82a0d9c832ecd namespace: kserve-ci-e2e-test resourceVersion: "15741" uid: 9bad61f2-d570-43ac-b226-d17ab1547281 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:18:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3bab1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15516" uid: 534a7c41-daa0-4f6d-ba73-6b808fd5b84b kind: Event lastTimestamp: "2026-04-20T20:19:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:18:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:19:26Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor.18a82a111b2898e0 namespace: kserve-ci-e2e-test resourceVersion: "16100" uid: 0cad496b-50ac-43d5-aea9-823889f8eec4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:18:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3bab1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15516" uid: 534a7c41-daa0-4f6d-ba73-6b808fd5b84b kind: Event lastTimestamp: "2026-04-20T20:19:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:18:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:19:26Z" name: isvc-xgboost-graph-raw-hpa-3bab1-predictor.18a82a111b84d132 namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: e48abd4b-783b-48c9-ac7a-857ad1369139 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-20T20:17:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-3bab1 namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: f1658fa6-2f02-442d-8691-526e5628a15c kind: Event lastTimestamp: "2026-04-20T20:18:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:18:58Z" name: isvc-xgboost-graph-raw-hpa-3bab1.18a82a0a1d831620 namespace: kserve-ci-e2e-test resourceVersion: "15876" uid: 76dc60c8-fbae-4c5e-ae13-f517916cb33c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-3bab1 namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: f1658fa6-2f02-442d-8691-526e5628a15c kind: Event lastTimestamp: "2026-04-20T20:17:55Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-3bab1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-3bab1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:17:55Z" name: isvc-xgboost-graph-raw-hpa-3bab1.18a82a0a1e05d370 namespace: kserve-ci-e2e-test resourceVersion: "15296" uid: 76d25562-939a-44c9-90c2-2a8b036983bd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:18:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-3bab1 namespace: kserve-ci-e2e-test resourceVersion: "15874" uid: f1658fa6-2f02-442d-8691-526e5628a15c kind: Event lastTimestamp: "2026-04-20T20:18:58Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-3bab1] is Ready metadata: creationTimestamp: "2026-04-20T20:18:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:18:58Z" name: isvc-xgboost-graph-raw-hpa-3bab1.18a82a189a480d42 namespace: kserve-ci-e2e-test resourceVersion: "15875" uid: b97d1027-3c4a-4341-b483-2fc2b5723039 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:19:36.237397Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn namespace: kserve-ci-e2e-test resourceVersion: "16246" uid: cae461df-647e-4cbd-bf0b-c5333d1c15eb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn to ip-10-0-135-184.ec2.internal metadata: creationTimestamp: "2026-04-20T20:19:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:19:36Z" name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn.18a82a21784003c3 namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: fbe4244c-c0ea-4ed7-be7c-8a98a3cc09bf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5766b54df6-m4bt5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:36Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: cae461df-647e-4cbd-bf0b-c5333d1c15eb kind: Event lastTimestamp: "2026-04-20T20:19:36Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:19:36Z" name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn.18a82a21916dfbcb namespace: kserve-ci-e2e-test resourceVersion: "16282" uid: e8bfb64f-25ed-4f03-a884-11d346bc10ca reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: cae461df-647e-4cbd-bf0b-c5333d1c15eb kind: Event lastTimestamp: "2026-04-20T20:19:36Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-04-20T20:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:36Z" name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn.18a82a2192b5a351 namespace: kserve-ci-e2e-test resourceVersion: "16284" uid: 123ec988-d80f-42f0-8143-87084c868448 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: cae461df-647e-4cbd-bf0b-c5333d1c15eb kind: Event lastTimestamp: "2026-04-20T20:19:37Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.066s (1.066s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-04-20T20:19:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:37Z" name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn.18a82a21d249c72b namespace: kserve-ci-e2e-test resourceVersion: "16292" uid: 77dd0d5c-3c18-40d0-b966-d61bdf24c854 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: cae461df-647e-4cbd-bf0b-c5333d1c15eb kind: Event lastTimestamp: "2026-04-20T20:19:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:19:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:37Z" name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn.18a82a21d53daafd namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: 19ca003f-5a2f-4fc8-9d01-92116aec99a5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: cae461df-647e-4cbd-bf0b-c5333d1c15eb kind: Event lastTimestamp: "2026-04-20T20:19:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:19:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:37Z" name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn.18a82a21d5b2dfee namespace: kserve-ci-e2e-test resourceVersion: "16294" uid: e4c873bd-094d-40c5-b386-a69a3e019e4e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: cae461df-647e-4cbd-bf0b-c5333d1c15eb kind: Event lastTimestamp: "2026-04-20T20:19:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:19:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:37Z" name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn.18a82a21d5be9432 namespace: kserve-ci-e2e-test resourceVersion: "16295" uid: b2a86f4f-4eb4-4e2e-a1ba-0ef656d68b53 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: cae461df-647e-4cbd-bf0b-c5333d1c15eb kind: Event lastTimestamp: "2026-04-20T20:19:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:19:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:37Z" name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn.18a82a21d8a12bcc namespace: kserve-ci-e2e-test resourceVersion: "16297" uid: d18086a7-aaca-4227-b9ca-e7de9755582d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: cae461df-647e-4cbd-bf0b-c5333d1c15eb kind: Event lastTimestamp: "2026-04-20T20:19:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:19:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:37Z" name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn.18a82a21d916b0ff namespace: kserve-ci-e2e-test resourceVersion: "16298" uid: de08031d-3132-4bb7-a969-b88353d12a83 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:21:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: cae461df-647e-4cbd-bf0b-c5333d1c15eb kind: Event lastTimestamp: "2026-04-20T20:21:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:21:21Z" name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn.18a82a39fccf3c89 namespace: kserve-ci-e2e-test resourceVersion: "17215" uid: b6d6b202-6efe-4c8c-825b-005030f6385b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:21:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn namespace: kserve-ci-e2e-test resourceVersion: "16250" uid: cae461df-647e-4cbd-bf0b-c5333d1c15eb kind: Event lastTimestamp: "2026-04-20T20:21:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:21:21Z" name: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn.18a82a39fccfb792 namespace: kserve-ci-e2e-test resourceVersion: "17216" uid: 5ae76166-d6d6-43ff-93ec-6440b1d6adb4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-5be0c-predictor-85d4fc8b75 namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 46dd8157-1fcf-4b56-9a21-91c024d26699 kind: Event lastTimestamp: "2026-04-20T20:19:36Z" message: 'Created pod: message-dumper-raw-5be0c-predictor-85d4fc8b75-sgjcn' metadata: creationTimestamp: "2026-04-20T20:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:19:36Z" name: message-dumper-raw-5be0c-predictor-85d4fc8b75.18a82a21779cdf53 namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: 8fc464f0-c86b-4f7d-9378-fdeac9533363 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-5be0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: fc50a632-ed62-422c-8af8-1ad3aa85f915 kind: Event lastTimestamp: "2026-04-20T20:19:36Z" message: Scaled up replica set message-dumper-raw-5be0c-predictor-85d4fc8b75 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:19:36Z" name: message-dumper-raw-5be0c-predictor.18a82a21728fbc5a namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 2f28f64b-de25-4860-a9f3-ad34b434610a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:19:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-5be0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16232" uid: 04606cce-ec53-4611-b3e5-e42442726330 kind: Event lastTimestamp: "2026-04-20T20:20:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:19:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:20:06Z" name: message-dumper-raw-5be0c-predictor.18a82a24f16e1c49 namespace: kserve-ci-e2e-test resourceVersion: "16596" uid: 6a4edf14-21cd-4004-b707-5e79850bdc8f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-20T20:19:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-5be0c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16232" uid: 04606cce-ec53-4611-b3e5-e42442726330 kind: Event lastTimestamp: "2026-04-20T20:20:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:19:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:20:06Z" name: message-dumper-raw-5be0c-predictor.18a82a24f1d7d495 namespace: kserve-ci-e2e-test resourceVersion: "16597" uid: 379dccc4-112e-408a-b06f-7ee9943ad55d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-20T20:19:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-5be0c namespace: kserve-ci-e2e-test resourceVersion: "16208" uid: 63fa6c56-a10e-4d83-84dd-8c5eccb816a3 kind: Event lastTimestamp: "2026-04-20T20:19:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:19:46Z" name: message-dumper-raw-5be0c.18a82a21726ef24d namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: 03b44a72-7966-42f6-89b3-114454d58baf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-5be0c namespace: kserve-ci-e2e-test resourceVersion: "16208" uid: 63fa6c56-a10e-4d83-84dd-8c5eccb816a3 kind: Event lastTimestamp: "2026-04-20T20:19:36Z" message: 'Failed to update status for InferenceService "message-dumper-raw-5be0c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-5be0c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:19:36Z" name: message-dumper-raw-5be0c.18a82a2172d8b6f1 namespace: kserve-ci-e2e-test resourceVersion: "16238" uid: 4e46e6c2-1c80-4611-94f2-f985e89eb016 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-5be0c namespace: kserve-ci-e2e-test resourceVersion: "16378" uid: 63fa6c56-a10e-4d83-84dd-8c5eccb816a3 kind: Event lastTimestamp: "2026-04-20T20:19:46Z" message: InferenceService [message-dumper-raw-5be0c] is Ready metadata: creationTimestamp: "2026-04-20T20:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:19:46Z" name: message-dumper-raw-5be0c.18a82a23db736690 namespace: kserve-ci-e2e-test resourceVersion: "16379" uid: 4e5a62b4-9655-4ad9-aa95-85dfff6c0ce2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:17:45.652577Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-87d08-7cf6cc6959-nllms namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: 93abaa13-1671-47af-8ad3-fa90ddd8fe04 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-87d08-7cf6cc6959-nllms to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-04-20T20:17:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:17:45Z" name: model-chainer-raw-87d08-7cf6cc6959-nllms.18a82a07b8e17f11 namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: 46b12e41-70d8-458d-8c3c-d9cf5c1912ce reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5766b54df6-m4bt5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:46Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-87d08-7cf6cc6959-nllms namespace: kserve-ci-e2e-test resourceVersion: "15116" uid: 93abaa13-1671-47af-8ad3-fa90ddd8fe04 kind: Event lastTimestamp: "2026-04-20T20:17:46Z" message: Add eth0 [10.132.0.16/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:17:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:17:46Z" name: model-chainer-raw-87d08-7cf6cc6959-nllms.18a82a07d1e3d133 namespace: kserve-ci-e2e-test resourceVersion: "15130" uid: cf4e75a1-1b86-4e98-bc77-8c542b471238 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-87d08} kind: Pod name: model-chainer-raw-87d08-7cf6cc6959-nllms namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 93abaa13-1671-47af-8ad3-fa90ddd8fe04 kind: Event lastTimestamp: "2026-04-20T20:17:46Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" metadata: creationTimestamp: "2026-04-20T20:17:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:46Z" name: model-chainer-raw-87d08-7cf6cc6959-nllms.18a82a07d329e004 namespace: kserve-ci-e2e-test resourceVersion: "15132" uid: 087da888-5e5e-434d-bd4a-d6713715b552 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-87d08} kind: Pod name: model-chainer-raw-87d08-7cf6cc6959-nllms namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 93abaa13-1671-47af-8ad3-fa90ddd8fe04 kind: Event lastTimestamp: "2026-04-20T20:17:48Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" in 2.474s (2.474s including waiting). Image size: 216233553 bytes.' metadata: creationTimestamp: "2026-04-20T20:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:48Z" name: model-chainer-raw-87d08-7cf6cc6959-nllms.18a82a0866a1a91f namespace: kserve-ci-e2e-test resourceVersion: "15151" uid: 67f9cb20-6975-47ef-9a95-65819c6f7710 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-87d08} kind: Pod name: model-chainer-raw-87d08-7cf6cc6959-nllms namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 93abaa13-1671-47af-8ad3-fa90ddd8fe04 kind: Event lastTimestamp: "2026-04-20T20:17:48Z" message: 'Created container: model-chainer-raw-87d08' metadata: creationTimestamp: "2026-04-20T20:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:48Z" name: model-chainer-raw-87d08-7cf6cc6959-nllms.18a82a086967a249 namespace: kserve-ci-e2e-test resourceVersion: "15152" uid: 0cbb9b8a-7ef3-449d-a803-1eb5a01fd4d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-87d08} kind: Pod name: model-chainer-raw-87d08-7cf6cc6959-nllms namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 93abaa13-1671-47af-8ad3-fa90ddd8fe04 kind: Event lastTimestamp: "2026-04-20T20:17:48Z" message: Started container model-chainer-raw-87d08 metadata: creationTimestamp: "2026-04-20T20:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:48Z" name: model-chainer-raw-87d08-7cf6cc6959-nllms.18a82a0869d06d8c namespace: kserve-ci-e2e-test resourceVersion: "15153" uid: 9daf1d35-29ab-4f7f-8d53-7566e2e049a8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-87d08} kind: Pod name: model-chainer-raw-87d08-7cf6cc6959-nllms namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 93abaa13-1671-47af-8ad3-fa90ddd8fe04 kind: Event lastTimestamp: "2026-04-20T20:17:55Z" message: Stopping container model-chainer-raw-87d08 metadata: creationTimestamp: "2026-04-20T20:17:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:17:55Z" name: model-chainer-raw-87d08-7cf6cc6959-nllms.18a82a0a10b12771 namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: b1b1e9a0-4589-4f7c-a71c-4cecb5226821 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:17:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-87d08} kind: Pod name: model-chainer-raw-87d08-7cf6cc6959-nllms namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 93abaa13-1671-47af-8ad3-fa90ddd8fe04 kind: Event lastTimestamp: "2026-04-20T20:18:24Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-20T20:17:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:18:24Z" name: model-chainer-raw-87d08-7cf6cc6959-nllms.18a82a0b0cd1060e namespace: kserve-ci-e2e-test resourceVersion: "15614" uid: 350d619c-be6b-41d9-a413-05a7261938a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-87d08-7cf6cc6959 namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: c245b8e8-6675-43b4-a297-c70791b1bfc2 kind: Event lastTimestamp: "2026-04-20T20:17:45Z" message: 'Created pod: model-chainer-raw-87d08-7cf6cc6959-nllms' metadata: creationTimestamp: "2026-04-20T20:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:17:45Z" name: model-chainer-raw-87d08-7cf6cc6959.18a82a07b8786760 namespace: kserve-ci-e2e-test resourceVersion: "15108" uid: 0d40c524-15ae-4e02-8cf0-facd1d14584a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-87d08 namespace: kserve-ci-e2e-test resourceVersion: "15102" uid: 6030810c-e395-448a-82c8-1288fd3e5e22 kind: Event lastTimestamp: "2026-04-20T20:17:45Z" message: Scaled up replica set model-chainer-raw-87d08-7cf6cc6959 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:17:45Z" name: model-chainer-raw-87d08.18a82a07b7b474b6 namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 40de3b8f-eba0-41e7-9a93-ac0aead4c7ed reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-87d08 namespace: kserve-ci-e2e-test resourceVersion: "15119" uid: 0e4e0c2b-11b4-49be-9477-6ee090b281c5 kind: Event lastTimestamp: "2026-04-20T20:17:45Z" message: InferenceGraph [model-chainer-raw-87d08] is Ready metadata: creationTimestamp: "2026-04-20T20:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:17:45Z" name: model-chainer-raw-87d08.18a82a07bcc48135 namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: 229656ef-0f12-42e7-9238-459a964ac749 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-87d08 namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: 0e4e0c2b-11b4-49be-9477-6ee090b281c5 kind: Event lastTimestamp: "2026-04-20T20:17:45Z" message: 'Failed to update status for InferenceGraph "model-chainer-raw-87d08": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-87d08": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:17:45Z" name: model-chainer-raw-87d08.18a82a07bee27368 namespace: kserve-ci-e2e-test resourceVersion: "15123" uid: e87fb0b4-cf3f-49c0-b3ac-5f1de34c957e reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:17:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-87d08 namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: 0e4e0c2b-11b4-49be-9477-6ee090b281c5 kind: Event lastTimestamp: "2026-04-20T20:17:45Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer-raw-87d08": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:17:45Z" name: model-chainer-raw-87d08.18a82a07bee2b40d namespace: kserve-ci-e2e-test resourceVersion: "15124" uid: 58ec4d0d-fd0c-4138-aa3d-049aeec67366 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:19:25.969905Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-3bab1-558467f979-8cb6f namespace: kserve-ci-e2e-test resourceVersion: "16089" uid: d90a3502-774b-42a3-9d44-d0c19127b044 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-3bab1-558467f979-8cb6f to ip-10-0-136-158.ec2.internal metadata: creationTimestamp: "2026-04-20T20:19:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:19:25Z" name: model-chainer-raw-hpa-3bab1-558467f979-8cb6f.18a82a1f1442756f namespace: kserve-ci-e2e-test resourceVersion: "16094" uid: ba2aa896-80f4-4112-a3b7-ced828b81942 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5766b54df6-m4bt5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:26Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-3bab1-558467f979-8cb6f namespace: kserve-ci-e2e-test resourceVersion: "16092" uid: d90a3502-774b-42a3-9d44-d0c19127b044 kind: Event lastTimestamp: "2026-04-20T20:19:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-3bab1-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:19:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:26Z" name: model-chainer-raw-hpa-3bab1-558467f979-8cb6f.18a82a1f2010edb6 namespace: kserve-ci-e2e-test resourceVersion: "16109" uid: 45ce286e-9d63-4bb2-bf5a-864e5c3707ee reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:27Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-3bab1-558467f979-8cb6f namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: d90a3502-774b-42a3-9d44-d0c19127b044 kind: Event lastTimestamp: "2026-04-20T20:19:27Z" message: Add eth0 [10.132.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:19:27Z" name: model-chainer-raw-hpa-3bab1-558467f979-8cb6f.18a82a1f51aaca9e namespace: kserve-ci-e2e-test resourceVersion: "16117" uid: 2fffdf19-6a0b-4a9b-8c7f-75921d0c178e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3bab1} kind: Pod name: model-chainer-raw-hpa-3bab1-558467f979-8cb6f namespace: kserve-ci-e2e-test resourceVersion: "16092" uid: d90a3502-774b-42a3-9d44-d0c19127b044 kind: Event lastTimestamp: "2026-04-20T20:19:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-20T20:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:27Z" name: model-chainer-raw-hpa-3bab1-558467f979-8cb6f.18a82a1f52f50cd6 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: 9df3c91b-871f-4c21-85f3-1dd6aed6a421 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3bab1} kind: Pod name: model-chainer-raw-hpa-3bab1-558467f979-8cb6f namespace: kserve-ci-e2e-test resourceVersion: "16092" uid: d90a3502-774b-42a3-9d44-d0c19127b044 kind: Event lastTimestamp: "2026-04-20T20:19:27Z" message: 'Created container: model-chainer-raw-hpa-3bab1' metadata: creationTimestamp: "2026-04-20T20:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:27Z" name: model-chainer-raw-hpa-3bab1-558467f979-8cb6f.18a82a1f564161f4 namespace: kserve-ci-e2e-test resourceVersion: "16120" uid: 82ad3b6f-09f9-4ed0-be34-5c641303721c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3bab1} kind: Pod name: model-chainer-raw-hpa-3bab1-558467f979-8cb6f namespace: kserve-ci-e2e-test resourceVersion: "16092" uid: d90a3502-774b-42a3-9d44-d0c19127b044 kind: Event lastTimestamp: "2026-04-20T20:19:27Z" message: Started container model-chainer-raw-hpa-3bab1 metadata: creationTimestamp: "2026-04-20T20:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:27Z" name: model-chainer-raw-hpa-3bab1-558467f979-8cb6f.18a82a1f56be1acd namespace: kserve-ci-e2e-test resourceVersion: "16121" uid: 3fb6b053-1e81-4361-9d71-fe3ac5f20ba6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3bab1} kind: Pod name: model-chainer-raw-hpa-3bab1-558467f979-8cb6f namespace: kserve-ci-e2e-test resourceVersion: "16092" uid: d90a3502-774b-42a3-9d44-d0c19127b044 kind: Event lastTimestamp: "2026-04-20T20:19:36Z" message: Stopping container model-chainer-raw-hpa-3bab1 metadata: creationTimestamp: "2026-04-20T20:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:19:36Z" name: model-chainer-raw-hpa-3bab1-558467f979-8cb6f.18a82a216aee70a3 namespace: kserve-ci-e2e-test resourceVersion: "16203" uid: 96a9edc1-0652-4658-acff-b08e60f039c3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-20T20:19:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3bab1} kind: Pod name: model-chainer-raw-hpa-3bab1-558467f979-8cb6f namespace: kserve-ci-e2e-test resourceVersion: "16092" uid: d90a3502-774b-42a3-9d44-d0c19127b044 kind: Event lastTimestamp: "2026-04-20T20:20:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-20T20:19:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:20:03Z" name: model-chainer-raw-hpa-3bab1-558467f979-8cb6f.18a82a21edd8f2f8 namespace: kserve-ci-e2e-test resourceVersion: "16569" uid: b6840737-0d3a-46f7-a628-6ed440bfee06 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-158.ec2.internal source: component: kubelet host: ip-10-0-136-158.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-3bab1-558467f979 namespace: kserve-ci-e2e-test resourceVersion: "16084" uid: 0743f99e-9b9f-42fd-aaf9-21a88d601bb8 kind: Event lastTimestamp: "2026-04-20T20:19:25Z" message: 'Created pod: model-chainer-raw-hpa-3bab1-558467f979-8cb6f' metadata: creationTimestamp: "2026-04-20T20:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:19:25Z" name: model-chainer-raw-hpa-3bab1-558467f979.18a82a1f13c438e9 namespace: kserve-ci-e2e-test resourceVersion: "16091" uid: 347ae02c-be31-4f6e-ae84-0c2b5474d96f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-3bab1 namespace: kserve-ci-e2e-test resourceVersion: "16082" uid: b39eeb3a-6763-45d6-8e3d-57ab077c47f1 kind: Event lastTimestamp: "2026-04-20T20:19:25Z" message: Scaled up replica set model-chainer-raw-hpa-3bab1-558467f979 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:19:25Z" name: model-chainer-raw-hpa-3bab1.18a82a1f1288f00c namespace: kserve-ci-e2e-test resourceVersion: "16085" uid: b5845149-e95a-4fd4-9da7-6d3645361201 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:19:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-3bab1 namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: 8342413e-4d13-451b-846d-9af62226e420 kind: Event lastTimestamp: "2026-04-20T20:19:26Z" message: InferenceGraph [model-chainer-raw-hpa-3bab1] is Ready metadata: creationTimestamp: "2026-04-20T20:19:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:19:26Z" name: model-chainer-raw-hpa-3bab1.18a82a1f1815641e namespace: kserve-ci-e2e-test resourceVersion: "16105" uid: be767f81-7fb7-4ffe-9d6d-4db25010d111 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:25:21.186982Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb namespace: kserve-ci-e2e-test resourceVersion: "19255" uid: 2b6adc77-1f4e-4265-8bc3-a61a9cabcfed kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb to ip-10-0-135-184.ec2.internal metadata: creationTimestamp: "2026-04-20T20:25:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:25:21Z" name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb.18a82a71c8d8e3eb namespace: kserve-ci-e2e-test resourceVersion: "19259" uid: 2f15a975-7e38-4053-aec8-9c7249d63810 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5766b54df6-m4bt5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:21Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb namespace: kserve-ci-e2e-test resourceVersion: "19265" uid: 2b6adc77-1f4e-4265-8bc3-a61a9cabcfed kind: Event lastTimestamp: "2026-04-20T20:25:21Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:25:21Z" name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb.18a82a71e27c0c84 namespace: kserve-ci-e2e-test resourceVersion: "19276" uid: c01fb571-2219-40a6-9f8c-7479ff3365ed reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: 2b6adc77-1f4e-4265-8bc3-a61a9cabcfed kind: Event lastTimestamp: "2026-04-20T20:25:21Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" already present on machine metadata: creationTimestamp: "2026-04-20T20:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:21Z" name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb.18a82a71e3b77a41 namespace: kserve-ci-e2e-test resourceVersion: "19278" uid: ef79169c-8d9c-4540-b9a0-504711d20fd5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: 2b6adc77-1f4e-4265-8bc3-a61a9cabcfed kind: Event lastTimestamp: "2026-04-20T20:25:21Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-20T20:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:21Z" name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb.18a82a71e75e7697 namespace: kserve-ci-e2e-test resourceVersion: "19279" uid: f703d2a2-bdf9-4c8d-9c6e-a7bdf16136fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: 2b6adc77-1f4e-4265-8bc3-a61a9cabcfed kind: Event lastTimestamp: "2026-04-20T20:25:21Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-20T20:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:21Z" name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb.18a82a71e7d5913a namespace: kserve-ci-e2e-test resourceVersion: "19280" uid: e9db6905-f7db-4912-ad6e-442728a11b38 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: 2b6adc77-1f4e-4265-8bc3-a61a9cabcfed kind: Event lastTimestamp: "2026-04-20T20:25:26Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:26Z" name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb.18a82a730cb621f4 namespace: kserve-ci-e2e-test resourceVersion: "19327" uid: f47ed167-6b45-4347-a820-ce480386dab2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: 2b6adc77-1f4e-4265-8bc3-a61a9cabcfed kind: Event lastTimestamp: "2026-04-20T20:25:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:26Z" name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb.18a82a730fda8549 namespace: kserve-ci-e2e-test resourceVersion: "19332" uid: 5b39583a-043c-48df-a1fe-a7f24e6576f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: 2b6adc77-1f4e-4265-8bc3-a61a9cabcfed kind: Event lastTimestamp: "2026-04-20T20:25:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:26Z" name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb.18a82a731054c747 namespace: kserve-ci-e2e-test resourceVersion: "19333" uid: af9451d5-c572-4f16-9f22-b7b3555e81f9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: 2b6adc77-1f4e-4265-8bc3-a61a9cabcfed kind: Event lastTimestamp: "2026-04-20T20:25:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:26Z" name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb.18a82a7310613ac6 namespace: kserve-ci-e2e-test resourceVersion: "19334" uid: 0f98ac7d-62b6-470f-9ef0-168e13b05e11 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: 2b6adc77-1f4e-4265-8bc3-a61a9cabcfed kind: Event lastTimestamp: "2026-04-20T20:25:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:26Z" name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb.18a82a731348e705 namespace: kserve-ci-e2e-test resourceVersion: "19336" uid: 2b0d74c7-c200-4c25-a529-721f4256617f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: 2b6adc77-1f4e-4265-8bc3-a61a9cabcfed kind: Event lastTimestamp: "2026-04-20T20:25:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:25:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:25:26Z" name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb.18a82a7313cedd02 namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: 2eed9c91-919d-4e7e-ae02-6d4b70509e0a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-20T20:25:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: 2b6adc77-1f4e-4265-8bc3-a61a9cabcfed kind: Event lastTimestamp: "2026-04-20T20:26:43Z" message: 'Readiness probe failed: dial tcp 10.133.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:25:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:43Z" name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb.18a82a7348ab7922 namespace: kserve-ci-e2e-test resourceVersion: "19956" uid: 13bae8b0-21b4-4ca7-a03e-ec08d3795a35 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: 2b6adc77-1f4e-4265-8bc3-a61a9cabcfed kind: Event lastTimestamp: "2026-04-20T20:26:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:41Z" name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb.18a82a8470e0aaf6 namespace: kserve-ci-e2e-test resourceVersion: "19898" uid: 57e4587a-6655-4f77-8b15-23f5dc4b917b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: 2b6adc77-1f4e-4265-8bc3-a61a9cabcfed kind: Event lastTimestamp: "2026-04-20T20:26:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:41Z" name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb.18a82a8470e28239 namespace: kserve-ci-e2e-test resourceVersion: "19903" uid: eb1b8038-a6a7-4810-be67-0ef6f471c51a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb namespace: kserve-ci-e2e-test resourceVersion: "19257" uid: 2b6adc77-1f4e-4265-8bc3-a61a9cabcfed kind: Event lastTimestamp: "2026-04-20T20:26:43Z" message: 'Readiness probe failed: Get "https://10.133.0.43:8643/healthz": dial tcp 10.133.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:26:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:43Z" name: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb.18a82a84facae364 namespace: kserve-ci-e2e-test resourceVersion: "19955" uid: 546dc136-30d5-4699-9712-95a319495bf9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-8db3a-predictor-5b77d9bc95 namespace: kserve-ci-e2e-test resourceVersion: "19245" uid: bb040c8a-56ec-4cbb-91d9-111a0eb67c32 kind: Event lastTimestamp: "2026-04-20T20:25:21Z" message: 'Created pod: raw-sklearn-8db3a-predictor-5b77d9bc95-2m5hb' metadata: creationTimestamp: "2026-04-20T20:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:21Z" name: raw-sklearn-8db3a-predictor-5b77d9bc95.18a82a71c8514a4b namespace: kserve-ci-e2e-test resourceVersion: "19256" uid: 0f658d80-42b8-41e0-8935-70c4ab43e2af reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-8db3a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19243" uid: 7fa05b31-afda-44ad-9fb3-45e3c0aa3722 kind: Event lastTimestamp: "2026-04-20T20:25:21Z" message: Scaled up replica set raw-sklearn-8db3a-predictor-5b77d9bc95 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:21Z" name: raw-sklearn-8db3a-predictor.18a82a71c5936465 namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: ba8a1994-daef-4f11-9b2d-6b7f0d87501f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-8db3a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19244" uid: cdb60a3b-9c27-492c-af68-a91ba41532fc kind: Event lastTimestamp: "2026-04-20T20:25:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:25:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:36Z" name: raw-sklearn-8db3a-predictor.18a82a7543fc7a89 namespace: kserve-ci-e2e-test resourceVersion: "19409" uid: 67ac3d71-be63-4e8d-b007-714d8ee7d096 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-8db3a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19244" uid: cdb60a3b-9c27-492c-af68-a91ba41532fc kind: Event lastTimestamp: "2026-04-20T20:25:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:25:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:25:36Z" name: raw-sklearn-8db3a-predictor.18a82a75445bfc7a namespace: kserve-ci-e2e-test resourceVersion: "19411" uid: b8b0cfd1-ce63-4f07-8e45-85bd29f77ffb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:25:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-8db3a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19410" uid: cdb60a3b-9c27-492c-af68-a91ba41532fc kind: Event lastTimestamp: "2026-04-20T20:26:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:25:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:26:36Z" name: raw-sklearn-8db3a-predictor.18a82a78c3808aa4 namespace: kserve-ci-e2e-test resourceVersion: "19835" uid: 97e32396-c740-47bd-86f0-bd4052128d93 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:25:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-8db3a-predictor namespace: kserve-ci-e2e-test resourceVersion: "19410" uid: cdb60a3b-9c27-492c-af68-a91ba41532fc kind: Event lastTimestamp: "2026-04-20T20:26:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:25:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:26:36Z" name: raw-sklearn-8db3a-predictor.18a82a78c3e622bf namespace: kserve-ci-e2e-test resourceVersion: "19837" uid: d14199f4-3a8c-462b-935a-9b0d5bed14e6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-20T20:25:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-8db3a namespace: kserve-ci-e2e-test resourceVersion: "19232" uid: ccd49b56-bbbe-4bf9-9c43-ea6e20da90c9 kind: Event lastTimestamp: "2026-04-20T20:26:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:26:33Z" name: raw-sklearn-8db3a.18a82a71c56e2b68 namespace: kserve-ci-e2e-test resourceVersion: "19818" uid: f8d79ce3-b1a5-4f1f-961a-19cd176b2a6b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:25:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-8db3a namespace: kserve-ci-e2e-test resourceVersion: "19232" uid: ccd49b56-bbbe-4bf9-9c43-ea6e20da90c9 kind: Event lastTimestamp: "2026-04-20T20:25:21Z" message: 'Failed to update status for InferenceService "raw-sklearn-8db3a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-8db3a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:25:21Z" name: raw-sklearn-8db3a.18a82a71c5bf7f0a namespace: kserve-ci-e2e-test resourceVersion: "19249" uid: a2f074e5-d145-412d-bb85-17cfbc5cd130 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-8db3a namespace: kserve-ci-e2e-test resourceVersion: "19815" uid: ccd49b56-bbbe-4bf9-9c43-ea6e20da90c9 kind: Event lastTimestamp: "2026-04-20T20:26:33Z" message: InferenceService [raw-sklearn-8db3a] is Ready metadata: creationTimestamp: "2026-04-20T20:26:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:26:33Z" name: raw-sklearn-8db3a.18a82a82ab6e30ef namespace: kserve-ci-e2e-test resourceVersion: "19816" uid: 8ef46571-3677-4bf5-9927-71fdd7205fce reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-20T20:26:41.381941Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54 namespace: kserve-ci-e2e-test resourceVersion: "19912" uid: b84ff881-c782-4982-9d8a-9af8c5ed94b5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54 to ip-10-0-135-184.ec2.internal metadata: creationTimestamp: "2026-04-20T20:26:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-20T20:26:41Z" name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54.18a82a8474d6e0a3 namespace: kserve-ci-e2e-test resourceVersion: "19917" uid: 40d8a005-a669-4028-b48e-17dbbbda855a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5766b54df6-m4bt5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:41Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54 namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: b84ff881-c782-4982-9d8a-9af8c5ed94b5 kind: Event lastTimestamp: "2026-04-20T20:26:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-runtime-146a8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-20T20:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:41Z" name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54.18a82a8485a3c234 namespace: kserve-ci-e2e-test resourceVersion: "19926" uid: 79e7acb4-ede0-431a-a3cb-7619600c8ac0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:42Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54 namespace: kserve-ci-e2e-test resourceVersion: "19921" uid: b84ff881-c782-4982-9d8a-9af8c5ed94b5 kind: Event lastTimestamp: "2026-04-20T20:26:42Z" message: Add eth0 [10.133.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-20T20:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-20T20:26:42Z" name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54.18a82a84b1dfdb51 namespace: kserve-ci-e2e-test resourceVersion: "19935" uid: 12d82977-24a2-4bc0-88b9-7c264a4e128c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54 namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: b84ff881-c782-4982-9d8a-9af8c5ed94b5 kind: Event lastTimestamp: "2026-04-20T20:26:42Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" already present on machine metadata: creationTimestamp: "2026-04-20T20:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:42Z" name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54.18a82a84b32d10a9 namespace: kserve-ci-e2e-test resourceVersion: "19937" uid: 3172ec38-23b6-40d6-8c64-4a23b43a1dfc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54 namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: b84ff881-c782-4982-9d8a-9af8c5ed94b5 kind: Event lastTimestamp: "2026-04-20T20:26:42Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-20T20:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:42Z" name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54.18a82a84b6446797 namespace: kserve-ci-e2e-test resourceVersion: "19939" uid: 53118358-e47d-4465-b416-a8641bc5892c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54 namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: b84ff881-c782-4982-9d8a-9af8c5ed94b5 kind: Event lastTimestamp: "2026-04-20T20:26:42Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-20T20:26:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:42Z" name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54.18a82a84b6aae4b3 namespace: kserve-ci-e2e-test resourceVersion: "19940" uid: 03b02266-8924-4e96-8486-b88cb200f056 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54 namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: b84ff881-c782-4982-9d8a-9af8c5ed94b5 kind: Event lastTimestamp: "2026-04-20T20:26:46Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-20T20:26:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:46Z" name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54.18a82a85be320265 namespace: kserve-ci-e2e-test resourceVersion: "19985" uid: a1b30b0a-cfad-4635-b624-58adcd2733ea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54 namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: b84ff881-c782-4982-9d8a-9af8c5ed94b5 kind: Event lastTimestamp: "2026-04-20T20:26:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-20T20:26:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:46Z" name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54.18a82a85c1138d13 namespace: kserve-ci-e2e-test resourceVersion: "19988" uid: 3d3796f6-d758-4d42-a9fd-85d200a03151 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54 namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: b84ff881-c782-4982-9d8a-9af8c5ed94b5 kind: Event lastTimestamp: "2026-04-20T20:26:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-20T20:26:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:46Z" name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54.18a82a85c1848d6a namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: c7e096c6-2e8c-4575-96f2-167e26d0610d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54 namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: b84ff881-c782-4982-9d8a-9af8c5ed94b5 kind: Event lastTimestamp: "2026-04-20T20:26:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-20T20:26:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:46Z" name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54.18a82a85c191bda9 namespace: kserve-ci-e2e-test resourceVersion: "19990" uid: 33f4a347-9b20-40ed-af7d-0a695df44cfa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54 namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: b84ff881-c782-4982-9d8a-9af8c5ed94b5 kind: Event lastTimestamp: "2026-04-20T20:26:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-20T20:26:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:47Z" name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54.18a82a85c579807d namespace: kserve-ci-e2e-test resourceVersion: "19993" uid: 2a71dd40-4a0a-42ef-8e63-b943672b312f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54 namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: b84ff881-c782-4982-9d8a-9af8c5ed94b5 kind: Event lastTimestamp: "2026-04-20T20:26:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:26:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:26:47Z" name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54.18a82a85c60588fe namespace: kserve-ci-e2e-test resourceVersion: "19994" uid: bf064a2e-e067-43e2-9b1c-17e31bb25ba9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-20T20:26:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54 namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: b84ff881-c782-4982-9d8a-9af8c5ed94b5 kind: Event lastTimestamp: "2026-04-20T20:28:04Z" message: 'Readiness probe failed: dial tcp 10.133.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:26:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:28:04Z" name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54.18a82a86360daeba namespace: kserve-ci-e2e-test resourceVersion: "20606" uid: d061652e-ade9-4029-ac3d-e6581e68c05d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:28:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54 namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: b84ff881-c782-4982-9d8a-9af8c5ed94b5 kind: Event lastTimestamp: "2026-04-20T20:28:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-20T20:28:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:28:01Z" name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54.18a82a971a0888b2 namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: a5c0bdb1-8291-40b9-a450-f34462f8553e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:28:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54 namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: b84ff881-c782-4982-9d8a-9af8c5ed94b5 kind: Event lastTimestamp: "2026-04-20T20:28:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-20T20:28:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:28:01Z" name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54.18a82a971a09623d namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: 6c7cf072-ccdc-4556-8ece-6175a46c0cb9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:28:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54 namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: b84ff881-c782-4982-9d8a-9af8c5ed94b5 kind: Event lastTimestamp: "2026-04-20T20:28:04Z" message: 'Readiness probe failed: Get "https://10.133.0.44:8643/healthz": dial tcp 10.133.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-20T20:28:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-20T20:28:04Z" name: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54.18a82a97e83125da namespace: kserve-ci-e2e-test resourceVersion: "20604" uid: 4c22bb3a-958d-4139-b879-fe59259df400 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-184.ec2.internal source: component: kubelet host: ip-10-0-135-184.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-146a8-predictor-57d4c64645 namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: 390f926f-fb1d-4141-8e04-779606bbb7ec kind: Event lastTimestamp: "2026-04-20T20:26:41Z" message: 'Created pod: raw-sklearn-runtime-146a8-predictor-57d4c64645-rvg54' metadata: creationTimestamp: "2026-04-20T20:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:26:41Z" name: raw-sklearn-runtime-146a8-predictor-57d4c64645.18a82a8474726ef5 namespace: kserve-ci-e2e-test resourceVersion: "19914" uid: 6deef4b7-8884-4683-886a-ec5c52563cc5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-146a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19896" uid: 9544a4e7-9a10-44c6-9d01-080b32d22a58 kind: Event lastTimestamp: "2026-04-20T20:26:41Z" message: Scaled up replica set raw-sklearn-runtime-146a8-predictor-57d4c64645 from 0 to 1 metadata: creationTimestamp: "2026-04-20T20:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:26:41Z" name: raw-sklearn-runtime-146a8-predictor.18a82a8471773bf1 namespace: kserve-ci-e2e-test resourceVersion: "19902" uid: 59753eda-3ac5-4560-99d1-c93ce655c588 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-146a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19897" uid: 2b45ab3c-d5ed-4b94-9932-fb032d37984c kind: Event lastTimestamp: "2026-04-20T20:26:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:26:56Z" name: raw-sklearn-runtime-146a8-predictor.18a82a87efe8cb4d namespace: kserve-ci-e2e-test resourceVersion: "20065" uid: 6c206722-9aab-4a8a-9c8d-9e75c30e58b7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-146a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "19897" uid: 2b45ab3c-d5ed-4b94-9932-fb032d37984c kind: Event lastTimestamp: "2026-04-20T20:26:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-20T20:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:26:56Z" name: raw-sklearn-runtime-146a8-predictor.18a82a87f048a52d namespace: kserve-ci-e2e-test resourceVersion: "20067" uid: 78f0d88b-68db-4680-b190-40252bedb590 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:27:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-146a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20066" uid: 2b45ab3c-d5ed-4b94-9932-fb032d37984c kind: Event lastTimestamp: "2026-04-20T20:27:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:27:56Z" name: raw-sklearn-runtime-146a8-predictor.18a82a8b6f6e9069 namespace: kserve-ci-e2e-test resourceVersion: "20494" uid: 20761a5c-d808-40e1-bbd2-9882b340cba8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-20T20:27:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-146a8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20066" uid: 2b45ab3c-d5ed-4b94-9932-fb032d37984c kind: Event lastTimestamp: "2026-04-20T20:27:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-20T20:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-20T20:27:56Z" name: raw-sklearn-runtime-146a8-predictor.18a82a8b6fd53d22 namespace: kserve-ci-e2e-test resourceVersion: "20495" uid: ce6d447e-99af-42cc-9f87-15869d1f61ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-20T20:26:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-146a8 namespace: kserve-ci-e2e-test resourceVersion: "19870" uid: 0059c338-157a-4f5b-80c3-928a4bb46656 kind: Event lastTimestamp: "2026-04-20T20:27:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-20T20:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:27:55Z" name: raw-sklearn-runtime-146a8.18a82a847148f2eb namespace: kserve-ci-e2e-test resourceVersion: "20480" uid: faaea8f8-7cdc-42a2-b25b-ab4563ee6a5f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:26:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-146a8 namespace: kserve-ci-e2e-test resourceVersion: "19870" uid: 0059c338-157a-4f5b-80c3-928a4bb46656 kind: Event lastTimestamp: "2026-04-20T20:26:41Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-146a8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-146a8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-20T20:26:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:26:41Z" name: raw-sklearn-runtime-146a8.18a82a8471d90993 namespace: kserve-ci-e2e-test resourceVersion: "19906" uid: 642c754c-4947-443e-b50b-6e9a34985812 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-20T20:27:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-146a8 namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: 0059c338-157a-4f5b-80c3-928a4bb46656 kind: Event lastTimestamp: "2026-04-20T20:27:55Z" message: InferenceService [raw-sklearn-runtime-146a8] is Ready metadata: creationTimestamp: "2026-04-20T20:27:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-20T20:27:55Z" name: raw-sklearn-runtime-146a8.18a82a959945ce14 namespace: kserve-ci-e2e-test resourceVersion: "20479" uid: 44000c48-bb5f-434b-ad5a-81d88b53738d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "20747"