--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:01:45.793394Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-491244-predictor-7c78c9fc8-wgp5f namespace: kserve-ci-e2e-test resourceVersion: "22281" uid: 9e3a8f33-eba7-449f-969d-a2bd1b73f5a7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-491244-predictor-7c78c9fc8-wgp5f to ip-10-0-142-94.ec2.internal metadata: creationTimestamp: "2026-07-08T17:01:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:01:45Z" name: isvc-init-fail-491244-predictor-7c78c9fc8-wgp5f.18c05f2e8207dfe0 namespace: kserve-ci-e2e-test resourceVersion: "22287" uid: 924e96cc-4933-4aab-9be9-065da8b7aabd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd6c76d55-gncps source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:01:46Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-491244-predictor-7c78c9fc8-wgp5f namespace: kserve-ci-e2e-test resourceVersion: "22291" uid: 9e3a8f33-eba7-449f-969d-a2bd1b73f5a7 kind: Event lastTimestamp: "2026-07-08T17:01:46Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:01:46Z" name: isvc-init-fail-491244-predictor-7c78c9fc8-wgp5f.18c05f2e9bdd56bf namespace: kserve-ci-e2e-test resourceVersion: "22300" uid: c9bad60f-5747-4e52-9d17-5db5f1b288ba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-491244-predictor-7c78c9fc8-wgp5f namespace: kserve-ci-e2e-test resourceVersion: "22283" uid: 9e3a8f33-eba7-449f-969d-a2bd1b73f5a7 kind: Event lastTimestamp: "2026-07-08T17:01:51Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7a9c075ae5e64d970cc026f71d8348f193fda0ed6de6807045e29c208821be1e" already present on machine metadata: creationTimestamp: "2026-07-08T17:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:01:51Z" name: isvc-init-fail-491244-predictor-7c78c9fc8-wgp5f.18c05f2e9d4b99d4 namespace: kserve-ci-e2e-test resourceVersion: "22355" uid: 88d15671-ab95-4da5-a304-a93bcca1aa72 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-491244-predictor-7c78c9fc8-wgp5f namespace: kserve-ci-e2e-test resourceVersion: "22283" uid: 9e3a8f33-eba7-449f-969d-a2bd1b73f5a7 kind: Event lastTimestamp: "2026-07-08T17:01:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T17:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:01:51Z" name: isvc-init-fail-491244-predictor-7c78c9fc8-wgp5f.18c05f2ea08f19ca namespace: kserve-ci-e2e-test resourceVersion: "22357" uid: 571567ca-7bde-4aa5-881f-02115e895457 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-491244-predictor-7c78c9fc8-wgp5f namespace: kserve-ci-e2e-test resourceVersion: "22283" uid: 9e3a8f33-eba7-449f-969d-a2bd1b73f5a7 kind: Event lastTimestamp: "2026-07-08T17:01:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T17:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:01:51Z" name: isvc-init-fail-491244-predictor-7c78c9fc8-wgp5f.18c05f2ea1098382 namespace: kserve-ci-e2e-test resourceVersion: "22358" uid: bf6c3356-b04b-4a1b-862d-15e36dec2b9a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:01:55Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-491244-predictor-7c78c9fc8-wgp5f namespace: kserve-ci-e2e-test resourceVersion: "22283" uid: 9e3a8f33-eba7-449f-969d-a2bd1b73f5a7 kind: Event lastTimestamp: "2026-07-08T17:01:55Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-07-08T17:01:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:01:55Z" name: isvc-init-fail-491244-predictor-7c78c9fc8-wgp5f.18c05f30d60b09bb namespace: kserve-ci-e2e-test resourceVersion: "22407" uid: b4a6b922-7858-4a6d-9f46-27169e45c881 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:01:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-491244-predictor-7c78c9fc8 namespace: kserve-ci-e2e-test resourceVersion: "22272" uid: d54a0802-40aa-4866-99af-5a515ba22326 kind: Event lastTimestamp: "2026-07-08T17:01:45Z" message: 'Created pod: isvc-init-fail-491244-predictor-7c78c9fc8-wgp5f' metadata: creationTimestamp: "2026-07-08T17:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:01:45Z" name: isvc-init-fail-491244-predictor-7c78c9fc8.18c05f2e818e8cb0 namespace: kserve-ci-e2e-test resourceVersion: "22282" uid: 25105a54-6021-47cb-b156-2604e8f28e88 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:01:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-491244-predictor namespace: kserve-ci-e2e-test resourceVersion: "22269" uid: 6a3f3de0-626f-411d-aef9-f58837b3c461 kind: Event lastTimestamp: "2026-07-08T17:01:45Z" message: Scaled up replica set isvc-init-fail-491244-predictor-7c78c9fc8 from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:01:45Z" name: isvc-init-fail-491244-predictor.18c05f2e7e9d7ec6 namespace: kserve-ci-e2e-test resourceVersion: "22273" uid: 172c0e32-48fb-4b10-ab72-cb42ca831935 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-08T17:01:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-491244 namespace: kserve-ci-e2e-test resourceVersion: "22262" uid: 0b14fcb3-8cdf-49af-9f08-28a027ace4c1 kind: Event lastTimestamp: "2026-07-08T17:01:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T17:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:01:51Z" name: isvc-init-fail-491244.18c05f2e7eb4b5b2 namespace: kserve-ci-e2e-test resourceVersion: "22362" uid: c03d881f-863c-4666-97fc-fbebb76ca918 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:01:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-491244 namespace: kserve-ci-e2e-test resourceVersion: "22262" uid: 0b14fcb3-8cdf-49af-9f08-28a027ace4c1 kind: Event lastTimestamp: "2026-07-08T17:01:45Z" message: 'Failed to update status for InferenceService "isvc-init-fail-491244": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-491244": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:01:45Z" name: isvc-init-fail-491244.18c05f2e7f1dd378 namespace: kserve-ci-e2e-test resourceVersion: "22276" uid: 1938c443-4695-4d4e-8cbd-635c03407144 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:01:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-491244 namespace: kserve-ci-e2e-test resourceVersion: "22262" uid: 0b14fcb3-8cdf-49af-9f08-28a027ace4c1 kind: Event lastTimestamp: "2026-07-08T17:01:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-491244": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:01:45Z" name: isvc-init-fail-491244.18c05f2e7f1e712e namespace: kserve-ci-e2e-test resourceVersion: "22277" uid: df4e88a8-d432-408e-9d2f-8d61c72c53e9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:55:33.600885Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19243" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-9f040-predictor-758d49c58-7gb6c to ip-10-0-142-94.ec2.internal metadata: creationTimestamp: "2026-07-08T16:55:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:55:33Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05ed7d9a0d43a namespace: kserve-ci-e2e-test resourceVersion: "19249" uid: 3bd1ff57-5bb8-485a-a642-ec95188d9057 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd6c76d55-gncps source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:33Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: "2026-07-08T16:55:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-raw-9f040-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:55:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:33Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05ed7e580d332 namespace: kserve-ci-e2e-test resourceVersion: "19256" uid: 2b8a7ba0-202b-491d-b0ac-f74e82ffdc4b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:34Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19253" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: "2026-07-08T16:55:34Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:55:34Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05ed8179f511b namespace: kserve-ci-e2e-test resourceVersion: "19266" uid: 1787dc09-eb9c-44af-8609-24534053d50e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: "2026-07-08T16:55:34Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7a9c075ae5e64d970cc026f71d8348f193fda0ed6de6807045e29c208821be1e" already present on machine metadata: creationTimestamp: "2026-07-08T16:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:34Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05ed8194f6c16 namespace: kserve-ci-e2e-test resourceVersion: "19269" uid: 9268f6ba-eb0c-4e10-9de8-90bedb8e50a0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: "2026-07-08T16:55:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:34Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05ed81d0d7a44 namespace: kserve-ci-e2e-test resourceVersion: "19270" uid: 5729f7fc-49d8-4249-8003-bf64f47a26da reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: "2026-07-08T16:55:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:55:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:34Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05ed81d910b65 namespace: kserve-ci-e2e-test resourceVersion: "19271" uid: f5c29916-6b61-4724-b778-bdd494d477f9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: "2026-07-08T16:55:39Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T16:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:39Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05ed92a3157dd namespace: kserve-ci-e2e-test resourceVersion: "19310" uid: 04b212d2-6d1f-43d2-9a71-42811ffac735 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: "2026-07-08T16:55:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:39Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05ed92d05ef4d namespace: kserve-ci-e2e-test resourceVersion: "19312" uid: faf6e7b4-6a4f-43be-b7f3-e2fdbe7d7e44 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: "2026-07-08T16:55:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:39Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05ed92d80df40 namespace: kserve-ci-e2e-test resourceVersion: "19313" uid: e4e0d519-6ca1-475e-97b6-ec34c3bbe3fd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: "2026-07-08T16:55:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:39Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05ed92d8c6b1e namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: 8ed8c255-100e-4fb9-b772-61d79c3c9fd6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: "2026-07-08T16:55:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:39Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05ed93130a020 namespace: kserve-ci-e2e-test resourceVersion: "19319" uid: f1becca3-c5d6-4899-afb0-42b2be3d243d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: "2026-07-08T16:55:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:39Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05ed931ad74dd namespace: kserve-ci-e2e-test resourceVersion: "19320" uid: ffbd0e64-add0-4358-a89f-334b0c3dfb9b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: "2026-07-08T16:55:39Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:dfb2082d89c1ab80bec06b84d0a6317b542c29ff4f60aa1cefe62d44619bad8a" already present on machine metadata: creationTimestamp: "2026-07-08T16:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:39Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05ed931b8d1b0 namespace: kserve-ci-e2e-test resourceVersion: "19321" uid: d7ecdb49-a9e6-4a4b-ad28-423ea1697ca2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: "2026-07-08T16:55:39Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-08T16:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:39Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05ed9348d9abb namespace: kserve-ci-e2e-test resourceVersion: "19322" uid: 05f3e354-bb4a-436e-8d42-aa0a9af627ad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: "2026-07-08T16:55:39Z" message: Started container agent metadata: creationTimestamp: "2026-07-08T16:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:39Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05ed93511cbf2 namespace: kserve-ci-e2e-test resourceVersion: "19324" uid: 352c528d-cc73-4703-b1bd-0f2b0e3a6afc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T16:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: "2026-07-08T16:57:16Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:16Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05ed96643967a namespace: kserve-ci-e2e-test resourceVersion: "20141" uid: 0175bd9e-90ab-48bc-af02-3b2d541b8036 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T16:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: "2026-07-08T16:57:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T16:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:16Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05ed9664f5728 namespace: kserve-ci-e2e-test resourceVersion: "20142" uid: 5a903066-5cc0-4809-90b4-aaa41276d41c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: "2026-07-08T16:57:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:57:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:08Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05eee0831bdc3 namespace: kserve-ci-e2e-test resourceVersion: "20025" uid: 3adadbed-f719-4aba-9bf5-0505be3d6c47 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: "2026-07-08T16:57:08Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-08T16:57:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:08Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05eee0831f28d namespace: kserve-ci-e2e-test resourceVersion: "20020" uid: daeb7c40-8d5b-4e8f-b3c1-b545e64f94f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: "2026-07-08T16:57:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:57:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:08Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05eee08326402 namespace: kserve-ci-e2e-test resourceVersion: "20030" uid: a28edd08-a86c-4be6-9f98-00a0680a3c55 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:57:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c namespace: kserve-ci-e2e-test resourceVersion: "19247" uid: 150db917-4ee1-488e-8070-c11d7057d3e1 kind: Event lastTimestamp: "2026-07-08T16:57:26Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:57:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:26Z" name: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c.18c05eee96809710 namespace: kserve-ci-e2e-test resourceVersion: "20218" uid: e0f36966-bd4a-4e73-9ab7-f1591c4cf10e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-9f040-predictor-758d49c58 namespace: kserve-ci-e2e-test resourceVersion: "19234" uid: 2bcd884b-b335-4399-8cb2-d9d559e61fc7 kind: Event lastTimestamp: "2026-07-08T16:55:33Z" message: 'Created pod: isvc-logger-raw-9f040-predictor-758d49c58-7gb6c' metadata: creationTimestamp: "2026-07-08T16:55:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:33Z" name: isvc-logger-raw-9f040-predictor-758d49c58.18c05ed7d9181ab6 namespace: kserve-ci-e2e-test resourceVersion: "19246" uid: 5d317fd8-7225-4ef0-b85e-0213f57a2be6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-9f040-predictor namespace: kserve-ci-e2e-test resourceVersion: "19232" uid: bdcdb5b9-cb60-4568-a6ac-c090c8ae20e1 kind: Event lastTimestamp: "2026-07-08T16:55:33Z" message: Scaled up replica set isvc-logger-raw-9f040-predictor-758d49c58 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:55:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:33Z" name: isvc-logger-raw-9f040-predictor.18c05ed7d60ae261 namespace: kserve-ci-e2e-test resourceVersion: "19235" uid: fb1be2b4-49f9-4d2b-a9cc-ff7c13962d65 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:55:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-9f040-predictor namespace: kserve-ci-e2e-test resourceVersion: "19233" uid: bf30c6cc-3879-47d4-8331-e4d15822ac63 kind: Event lastTimestamp: "2026-07-08T16:56:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:03Z" name: isvc-logger-raw-9f040-predictor.18c05edb548a0229 namespace: kserve-ci-e2e-test resourceVersion: "19510" uid: ac511368-6926-458c-acfc-d1c50b5bb512 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:55:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-9f040-predictor namespace: kserve-ci-e2e-test resourceVersion: "19233" uid: bf30c6cc-3879-47d4-8331-e4d15822ac63 kind: Event lastTimestamp: "2026-07-08T16:56:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:56:03Z" name: isvc-logger-raw-9f040-predictor.18c05edb54ed7aac namespace: kserve-ci-e2e-test resourceVersion: "19511" uid: 384c179f-f3bc-4444-bc37-74282e306600 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:56:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-9f040-predictor namespace: kserve-ci-e2e-test resourceVersion: "19400" uid: bf30c6cc-3879-47d4-8331-e4d15822ac63 kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:57:03Z" name: isvc-logger-raw-9f040-predictor.18c05ee2525d09d3 namespace: kserve-ci-e2e-test resourceVersion: "19949" uid: bad1f502-db16-45ab-ad26-66d4c2466d7b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:56:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-9f040-predictor namespace: kserve-ci-e2e-test resourceVersion: "19400" uid: bf30c6cc-3879-47d4-8331-e4d15822ac63 kind: Event lastTimestamp: "2026-07-08T16:57:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:56:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:57:03Z" name: isvc-logger-raw-9f040-predictor.18c05ee252c4fe19 namespace: kserve-ci-e2e-test resourceVersion: "19950" uid: b369c2b4-8b27-42b3-868d-38b2a67d83bc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-08T16:55:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-9f040 namespace: kserve-ci-e2e-test resourceVersion: "19225" uid: ad87e726-dcdb-47c4-b02a-4ecdbe3360d8 kind: Event lastTimestamp: "2026-07-08T16:56:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:55:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:56:56Z" name: isvc-logger-raw-9f040.18c05ed7d613691c namespace: kserve-ci-e2e-test resourceVersion: "19897" uid: 3412117f-0ae7-4c7d-8a6c-97ed0cd49bfe reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-9f040 namespace: kserve-ci-e2e-test resourceVersion: "19225" uid: ad87e726-dcdb-47c4-b02a-4ecdbe3360d8 kind: Event lastTimestamp: "2026-07-08T16:55:33Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-9f040": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-9f040": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:55:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:55:33Z" name: isvc-logger-raw-9f040.18c05ed7d6857adb namespace: kserve-ci-e2e-test resourceVersion: "19238" uid: 9ac7ecba-4c30-4419-9d7a-320fbeed8a53 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:56:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-9f040 namespace: kserve-ci-e2e-test resourceVersion: "19895" uid: ad87e726-dcdb-47c4-b02a-4ecdbe3360d8 kind: Event lastTimestamp: "2026-07-08T16:56:56Z" message: InferenceService [isvc-logger-raw-9f040] is Ready metadata: creationTimestamp: "2026-07-08T16:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:56:56Z" name: isvc-logger-raw-9f040.18c05eeb1fc41e57 namespace: kserve-ci-e2e-test resourceVersion: "19896" uid: 8ad8007a-ce7b-4306-8158-4977d4634b53 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:59:54.299023Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: 612009f3-1621-48f9-a0aa-838e07da4734 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj to ip-10-0-142-94.ec2.internal metadata: creationTimestamp: "2026-07-08T16:59:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:59:54Z" name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj.18c05f148c72b77e namespace: kserve-ci-e2e-test resourceVersion: "21312" uid: fe3d35f8-ef75-4326-8142-e17284272bc3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd6c76d55-gncps source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:59:54Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 612009f3-1621-48f9-a0aa-838e07da4734 kind: Event lastTimestamp: "2026-07-08T16:59:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-primary-ff3865-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:55Z" name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj.18c05f149870a4bb namespace: kserve-ci-e2e-test resourceVersion: "21328" uid: cbbc578e-1fb5-4192-98a5-557d4b39fb9f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:56Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj namespace: kserve-ci-e2e-test resourceVersion: "21316" uid: 612009f3-1621-48f9-a0aa-838e07da4734 kind: Event lastTimestamp: "2026-07-08T16:59:56Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:59:56Z" name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj.18c05f14ffa4f998 namespace: kserve-ci-e2e-test resourceVersion: "21339" uid: 051f07c1-c4d2-4aee-ad81-62aba04f0b6c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 612009f3-1621-48f9-a0aa-838e07da4734 kind: Event lastTimestamp: "2026-07-08T16:59:56Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7a9c075ae5e64d970cc026f71d8348f193fda0ed6de6807045e29c208821be1e" already present on machine metadata: creationTimestamp: "2026-07-08T16:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:56Z" name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj.18c05f15012e9ec6 namespace: kserve-ci-e2e-test resourceVersion: "21342" uid: bd997eb0-4d94-44c0-b441-e7f185e50e9f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 612009f3-1621-48f9-a0aa-838e07da4734 kind: Event lastTimestamp: "2026-07-08T16:59:56Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:56Z" name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj.18c05f15043bb0ec namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 56f7264c-2ff2-4042-8504-c77e8330438f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:56Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 612009f3-1621-48f9-a0aa-838e07da4734 kind: Event lastTimestamp: "2026-07-08T16:59:56Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:59:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:56Z" name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj.18c05f1504b88bd9 namespace: kserve-ci-e2e-test resourceVersion: "21345" uid: d0e21a32-0c62-4ebd-bcf2-98d4e032d7fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:00:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 612009f3-1621-48f9-a0aa-838e07da4734 kind: Event lastTimestamp: "2026-07-08T17:00:01Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T17:00:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:00:01Z" name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj.18c05f162f55c924 namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: 9297bc21-f06c-43b3-abf4-1797f0f9da38 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:00:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 612009f3-1621-48f9-a0aa-838e07da4734 kind: Event lastTimestamp: "2026-07-08T17:00:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T17:00:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:00:01Z" name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj.18c05f1632273a5e namespace: kserve-ci-e2e-test resourceVersion: "21390" uid: b9e03dcc-d9d7-48f1-9dca-b83a4e437591 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:00:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 612009f3-1621-48f9-a0aa-838e07da4734 kind: Event lastTimestamp: "2026-07-08T17:00:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T17:00:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:00:01Z" name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj.18c05f1632a2d0c2 namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: b459834a-82a4-419b-9354-c3c63db1bf3a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:00:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 612009f3-1621-48f9-a0aa-838e07da4734 kind: Event lastTimestamp: "2026-07-08T17:00:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T17:00:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:00:01Z" name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj.18c05f1632add158 namespace: kserve-ci-e2e-test resourceVersion: "21392" uid: b950ff97-1804-4dad-ad66-b933ff65c3fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:00:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 612009f3-1621-48f9-a0aa-838e07da4734 kind: Event lastTimestamp: "2026-07-08T17:00:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T17:00:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:00:01Z" name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj.18c05f1635a76ce9 namespace: kserve-ci-e2e-test resourceVersion: "21395" uid: 3fb132d4-4c75-4cba-a09e-2a31733fbce7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:00:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 612009f3-1621-48f9-a0aa-838e07da4734 kind: Event lastTimestamp: "2026-07-08T17:00:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:00:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:00:01Z" name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj.18c05f16361ce817 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: 20e36d04-c532-4520-b1c6-0a6662becc43 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-08T17:00:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 612009f3-1621-48f9-a0aa-838e07da4734 kind: Event lastTimestamp: "2026-07-08T17:01:09Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:00:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:01:09Z" name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj.18c05f16a72142e1 namespace: kserve-ci-e2e-test resourceVersion: "21883" uid: cc3a05d9-1d39-4e71-848b-caf0db7fe32b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:01:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 612009f3-1621-48f9-a0aa-838e07da4734 kind: Event lastTimestamp: "2026-07-08T17:01:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:01:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:01:40Z" name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj.18c05f2d4943ae4d namespace: kserve-ci-e2e-test resourceVersion: "22198" uid: 97edbabd-4913-4c2a-bb74-06ccd09bcd1e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:01:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 612009f3-1621-48f9-a0aa-838e07da4734 kind: Event lastTimestamp: "2026-07-08T17:01:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:01:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:01:40Z" name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj.18c05f2d49442635 namespace: kserve-ci-e2e-test resourceVersion: "22199" uid: b8d34f5c-ad57-4058-b080-d46896941b39 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:01:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 612009f3-1621-48f9-a0aa-838e07da4734 kind: Event lastTimestamp: "2026-07-08T17:01:44Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:01:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:01:44Z" name: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj.18c05f2e2b643856 namespace: kserve-ci-e2e-test resourceVersion: "22243" uid: 5bc1e6a4-eb43-455a-8ffd-acdf1e090f70 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-ff3865-predictor-6c5bfbff9f namespace: kserve-ci-e2e-test resourceVersion: "21297" uid: 3c63b03e-37a1-48ff-9013-46d66f85174c kind: Event lastTimestamp: "2026-07-08T16:59:54Z" message: 'Created pod: isvc-primary-ff3865-predictor-6c5bfbff9f-jspsj' metadata: creationTimestamp: "2026-07-08T16:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:54Z" name: isvc-primary-ff3865-predictor-6c5bfbff9f.18c05f148c062370 namespace: kserve-ci-e2e-test resourceVersion: "21309" uid: b2f16bb5-1379-440a-8e62-2fffc506cb3a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-ff3865-predictor namespace: kserve-ci-e2e-test resourceVersion: "21295" uid: 1475a802-bb56-4477-ab4b-d39b1735ad66 kind: Event lastTimestamp: "2026-07-08T16:59:54Z" message: Scaled up replica set isvc-primary-ff3865-predictor-6c5bfbff9f from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:54Z" name: isvc-primary-ff3865-predictor.18c05f14879d3542 namespace: kserve-ci-e2e-test resourceVersion: "21298" uid: 61ae5900-2abe-49e0-af45-9fde429016e0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:00:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-ff3865-predictor namespace: kserve-ci-e2e-test resourceVersion: "21296" uid: 4de6c7a4-57c6-48f9-a274-a07ceecd1786 kind: Event lastTimestamp: "2026-07-08T17:00:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:00:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:00:39Z" name: isvc-primary-ff3865-predictor.18c05f180657c4f0 namespace: kserve-ci-e2e-test resourceVersion: "21670" uid: 7d475bb5-546d-4c02-9257-427eabca882d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:00:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-ff3865-predictor namespace: kserve-ci-e2e-test resourceVersion: "21296" uid: 4de6c7a4-57c6-48f9-a274-a07ceecd1786 kind: Event lastTimestamp: "2026-07-08T17:00:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:00:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:00:39Z" name: isvc-primary-ff3865-predictor.18c05f1806c56d36 namespace: kserve-ci-e2e-test resourceVersion: "21671" uid: f86021ab-6ed8-4665-a016-5e5121a5e120 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T17:00:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-ff3865-predictor namespace: kserve-ci-e2e-test resourceVersion: "21461" uid: 4de6c7a4-57c6-48f9-a274-a07ceecd1786 kind: Event lastTimestamp: "2026-07-08T17:01:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:01:39Z" name: isvc-primary-ff3865-predictor.18c05f1b85deb4dc namespace: kserve-ci-e2e-test resourceVersion: "22166" uid: c88fc0b5-888f-413f-8de9-1abaf3e3a498 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T17:00:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-ff3865-predictor namespace: kserve-ci-e2e-test resourceVersion: "21461" uid: 4de6c7a4-57c6-48f9-a274-a07ceecd1786 kind: Event lastTimestamp: "2026-07-08T17:01:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:01:39Z" name: isvc-primary-ff3865-predictor.18c05f1b86415c62 namespace: kserve-ci-e2e-test resourceVersion: "22168" uid: 3009241d-fe3d-4fff-887f-da069aabbd84 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-08T16:59:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-ff3865 namespace: kserve-ci-e2e-test resourceVersion: "21288" uid: 1d50cdf9-8744-4948-b292-db85802dece2 kind: Event lastTimestamp: "2026-07-08T17:01:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:01:19Z" name: isvc-primary-ff3865.18c05f1487a9ea43 namespace: kserve-ci-e2e-test resourceVersion: "21958" uid: 7937eba7-85d3-4bb3-a880-57df9c420eab reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:59:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-ff3865 namespace: kserve-ci-e2e-test resourceVersion: "21288" uid: 1d50cdf9-8744-4948-b292-db85802dece2 kind: Event lastTimestamp: "2026-07-08T16:59:54Z" message: 'Failed to update status for InferenceService "isvc-primary-ff3865": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-ff3865": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:59:54Z" name: isvc-primary-ff3865.18c05f148865f9a3 namespace: kserve-ci-e2e-test resourceVersion: "21319" uid: 595f2245-7c7f-43c1-bb84-6df1a269e20d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:59:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-ff3865 namespace: kserve-ci-e2e-test resourceVersion: "21288" uid: 1d50cdf9-8744-4948-b292-db85802dece2 kind: Event lastTimestamp: "2026-07-08T16:59:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-ff3865": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:59:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:59:54Z" name: isvc-primary-ff3865.18c05f1488664955 namespace: kserve-ci-e2e-test resourceVersion: "21320" uid: 44958d25-6955-4337-80bd-7ee1dd76a398 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:01:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-ff3865 namespace: kserve-ci-e2e-test resourceVersion: "21955" uid: 1d50cdf9-8744-4948-b292-db85802dece2 kind: Event lastTimestamp: "2026-07-08T17:01:19Z" message: InferenceService [isvc-primary-ff3865] is Ready metadata: creationTimestamp: "2026-07-08T17:01:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:01:19Z" name: isvc-primary-ff3865.18c05f2860a29928 namespace: kserve-ci-e2e-test resourceVersion: "21957" uid: 545da13d-9f4c-4adf-b986-e001b42ba571 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:49:47.664258Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16109" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 to ip-10-0-142-94.ec2.internal metadata: creationTimestamp: "2026-07-08T16:49:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:49:47Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05e874e32d82c namespace: kserve-ci-e2e-test resourceVersion: "16113" uid: d2dc6b2e-0739-4bf6-87a5-e8bf1b293f71 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd6c76d55-gncps source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:48Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:49:48Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:49:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:49:48Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05e87686de260 namespace: kserve-ci-e2e-test resourceVersion: "16125" uid: a46505b7-ceb0-4684-a6b1-e5d797df797a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:49:48Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7a9c075ae5e64d970cc026f71d8348f193fda0ed6de6807045e29c208821be1e" already present on machine metadata: creationTimestamp: "2026-07-08T16:49:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:49:48Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05e8769ed37eb namespace: kserve-ci-e2e-test resourceVersion: "16128" uid: e343e8de-e327-4fec-bc9d-67d2cd3bc23f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:49:48Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:49:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:49:48Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05e876e44a128 namespace: kserve-ci-e2e-test resourceVersion: "16130" uid: 7b18208e-df03-4384-b491-96e653982aa4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:49:48Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:49:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:49:48Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05e876ec25fb7 namespace: kserve-ci-e2e-test resourceVersion: "16131" uid: 637f3a63-c478-4262-9955-b6752c432aea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:49:52Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1711" metadata: creationTimestamp: "2026-07-08T16:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:49:52Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05e887c5bd744 namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: 9c5dc8b5-c6c5-4bef-a289-f8e7a3cb89e6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:50:07Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1711" in 14.391s (14.391s including waiting). Image size: 1555863462 bytes.' metadata: creationTimestamp: "2026-07-08T16:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:07Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05e8bd62f585c namespace: kserve-ci-e2e-test resourceVersion: "16268" uid: e1682f16-3e2d-4d39-9a6e-dcdc498dd307 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:50:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:07Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05e8bd934858c namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 0b35526a-5816-43a6-bc74-28c0b7afc5dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:50:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:07Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05e8bd9b6fbf7 namespace: kserve-ci-e2e-test resourceVersion: "16271" uid: b861989f-87c5-4b9c-91d6-4ed232065133 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:50:07Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-08T16:50:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:07Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05e8bd9c7deed namespace: kserve-ci-e2e-test resourceVersion: "16272" uid: 704e5424-5582-4447-8229-f21054f8be34 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:50:10Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.315s (3.315s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-08T16:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:10Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05e8c9f678806 namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 6ea6682f-0f29-4dca-8011-2ea1bd255450 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:50:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:10Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05e8ca2503a56 namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 8b5c7253-6f3c-43f0-9249-d465439377a1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:50:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:10Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05e8ca2cadc46 namespace: kserve-ci-e2e-test resourceVersion: "16301" uid: 3ade3f6d-031a-45af-96a5-38884cb3bf54 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:50:10Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:dfb2082d89c1ab80bec06b84d0a6317b542c29ff4f60aa1cefe62d44619bad8a" metadata: creationTimestamp: "2026-07-08T16:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:10Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05e8ca2d98868 namespace: kserve-ci-e2e-test resourceVersion: "16302" uid: e558f1f2-5263-481f-b847-f6a7dd3e93eb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:50:13Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:dfb2082d89c1ab80bec06b84d0a6317b542c29ff4f60aa1cefe62d44619bad8a" in 3.122s (3.122s including waiting). Image size: 239997024 bytes.' metadata: creationTimestamp: "2026-07-08T16:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:13Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05e8d5cfacaa4 namespace: kserve-ci-e2e-test resourceVersion: "16323" uid: 110c273e-9d4b-416b-8c4f-84c09adb93d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:50:13Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-08T16:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:13Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05e8d607d80c8 namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: 09bfe35d-4345-4710-bde3-7d481f8e3af3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:50:13Z" message: Started container agent metadata: creationTimestamp: "2026-07-08T16:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:50:13Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05e8d6113678f namespace: kserve-ci-e2e-test resourceVersion: "16326" uid: 596402bb-e4ac-44b8-83b4-8b3d78bd0a27 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-08T16:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:51:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:59Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05e8d66bfbb50 namespace: kserve-ci-e2e-test resourceVersion: "17237" uid: c309fd3f-6853-4150-84f5-a4d9098131a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T16:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:51:49Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T16:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:49Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05e8d66c8ece2 namespace: kserve-ci-e2e-test resourceVersion: "17152" uid: 7059f12d-1d89-4941-aaf6-7980292e7966 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:51:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:42Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05ea2179618dd namespace: kserve-ci-e2e-test resourceVersion: "16989" uid: b71af178-7d6d-490d-91cd-4c373dc5ee56 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:51:42Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-08T16:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:42Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05ea21796690b namespace: kserve-ci-e2e-test resourceVersion: "16990" uid: 72641c51-b8ed-4e3d-85e3-8528005aa400 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:51:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:42Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05ea21796cf1a namespace: kserve-ci-e2e-test resourceVersion: "16991" uid: 08d21c09-d532-44cd-a79d-369ca26d3aca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:51:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: 2d9691d9-c2fa-42de-8f3b-4851317a6bac kind: Event lastTimestamp: "2026-07-08T16:51:59Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:51:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:59Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46.18c05ea296f25a48 namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: 3c4c0a5b-c7f4-4e27-8396-72aa4a8b30d2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d namespace: kserve-ci-e2e-test resourceVersion: "16095" uid: 0f3a7016-a63b-45df-a90d-9a6ebfda4d2b kind: Event lastTimestamp: "2026-07-08T16:49:47Z" message: 'Created pod: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d-qss46' metadata: creationTimestamp: "2026-07-08T16:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:49:47Z" name: isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d.18c05e874dcd4887 namespace: kserve-ci-e2e-test resourceVersion: "16110" uid: 42fcbf7c-331a-49b3-a5ed-50940e48fc75 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-03e1d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16094" uid: 6a7ba95a-857c-41e1-b159-cd097718029c kind: Event lastTimestamp: "2026-07-08T16:49:47Z" message: Scaled up replica set isvc-raw-sklearn-batcher-03e1d-predictor-7c6b878b5d from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:49:47Z" name: isvc-raw-sklearn-batcher-03e1d-predictor.18c05e8745a32728 namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: 6ef109f6-56a1-4df9-9778-7b419ca36f82 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:50:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-03e1d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16101" uid: 90484876-3647-443c-bd3e-ccfcad107405 kind: Event lastTimestamp: "2026-07-08T16:50:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:50:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:50:47Z" name: isvc-raw-sklearn-batcher-03e1d-predictor.18c05e8acafaaf6d namespace: kserve-ci-e2e-test resourceVersion: "16569" uid: 26242fa3-6a5d-43ae-8f16-ed4435cc85d8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:50:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-03e1d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16101" uid: 90484876-3647-443c-bd3e-ccfcad107405 kind: Event lastTimestamp: "2026-07-08T16:50:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:50:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:50:47Z" name: isvc-raw-sklearn-batcher-03e1d-predictor.18c05e8acb5dd610 namespace: kserve-ci-e2e-test resourceVersion: "16570" uid: 4994d6ad-988f-49e4-b802-52cc52c11ef1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:51:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-03e1d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 90484876-3647-443c-bd3e-ccfcad107405 kind: Event lastTimestamp: "2026-07-08T16:51:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:51:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:51:32Z" name: isvc-raw-sklearn-batcher-03e1d-predictor.18c05e98c72fad3e namespace: kserve-ci-e2e-test resourceVersion: "16901" uid: dbe30331-994b-4910-bd61-707772b3026d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:51:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-03e1d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 90484876-3647-443c-bd3e-ccfcad107405 kind: Event lastTimestamp: "2026-07-08T16:51:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:51:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:51:32Z" name: isvc-raw-sklearn-batcher-03e1d-predictor.18c05e98c7983041 namespace: kserve-ci-e2e-test resourceVersion: "16902" uid: 4e14e76d-4db5-40d5-b7cc-892f87bd4d47 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T16:49:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-03e1d namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 45779d70-ce64-4d9b-9245-a2af5c6b4bf5 kind: Event lastTimestamp: "2026-07-08T16:51:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:51:30Z" name: isvc-raw-sklearn-batcher-03e1d.18c05e874bc8b1da namespace: kserve-ci-e2e-test resourceVersion: "16881" uid: e6aa3603-a64e-4150-809f-1b003642df86 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-03e1d namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 45779d70-ce64-4d9b-9245-a2af5c6b4bf5 kind: Event lastTimestamp: "2026-07-08T16:49:47Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-03e1d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-03e1d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:49:47Z" name: isvc-raw-sklearn-batcher-03e1d.18c05e874c1b2056 namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: 2d7a6cec-201b-4bbd-874e-444024ec46de reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:49:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-03e1d namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 45779d70-ce64-4d9b-9245-a2af5c6b4bf5 kind: Event lastTimestamp: "2026-07-08T16:49:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-03e1d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:49:47Z" name: isvc-raw-sklearn-batcher-03e1d.18c05e874c1b7c9b namespace: kserve-ci-e2e-test resourceVersion: "16106" uid: 1fd2f505-9ed0-4337-9915-3af106539345 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-03e1d namespace: kserve-ci-e2e-test resourceVersion: "16878" uid: 45779d70-ce64-4d9b-9245-a2af5c6b4bf5 kind: Event lastTimestamp: "2026-07-08T16:51:29Z" message: InferenceService [isvc-raw-sklearn-batcher-03e1d] is Ready metadata: creationTimestamp: "2026-07-08T16:51:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:51:29Z" name: isvc-raw-sklearn-batcher-03e1d.18c05e9f20271fa2 namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: 19d9959c-104d-4cd4-9053-4a70297d3747 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:01:24.530787Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-ff3865-predictor-6b98f7f6fb-krwgl namespace: kserve-ci-e2e-test resourceVersion: "22026" uid: db0b0dfb-c87f-4735-b90f-558a56c7df01 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-ff3865-predictor-6b98f7f6fb-krwgl to ip-10-0-142-94.ec2.internal metadata: creationTimestamp: "2026-07-08T17:01:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:01:24Z" name: isvc-secondary-ff3865-predictor-6b98f7f6fb-krwgl.18c05f298eae473c namespace: kserve-ci-e2e-test resourceVersion: "22030" uid: e3d4a9d7-15ca-4190-bd77-0d20576ebc43 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd6c76d55-gncps source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:01:24Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-ff3865-predictor-6b98f7f6fb-krwgl namespace: kserve-ci-e2e-test resourceVersion: "22027" uid: db0b0dfb-c87f-4735-b90f-558a56c7df01 kind: Event lastTimestamp: "2026-07-08T17:01:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-secondary-ff3865-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T17:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:01:24Z" name: isvc-secondary-ff3865-predictor-6b98f7f6fb-krwgl.18c05f299d474560 namespace: kserve-ci-e2e-test resourceVersion: "22043" uid: 722b2087-7760-45ea-a933-19cb9f804c14 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:01:25Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-ff3865-predictor-6b98f7f6fb-krwgl namespace: kserve-ci-e2e-test resourceVersion: "22034" uid: db0b0dfb-c87f-4735-b90f-558a56c7df01 kind: Event lastTimestamp: "2026-07-08T17:01:25Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:01:25Z" name: isvc-secondary-ff3865-predictor-6b98f7f6fb-krwgl.18c05f29cbdef18a namespace: kserve-ci-e2e-test resourceVersion: "22052" uid: af3dfa60-9293-4637-9eb5-c89c37753db0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:01:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-ff3865-predictor-6b98f7f6fb-krwgl namespace: kserve-ci-e2e-test resourceVersion: "22027" uid: db0b0dfb-c87f-4735-b90f-558a56c7df01 kind: Event lastTimestamp: "2026-07-08T17:01:28Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7a9c075ae5e64d970cc026f71d8348f193fda0ed6de6807045e29c208821be1e" already present on machine metadata: creationTimestamp: "2026-07-08T17:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:01:28Z" name: isvc-secondary-ff3865-predictor-6b98f7f6fb-krwgl.18c05f29cd5f75b4 namespace: kserve-ci-e2e-test resourceVersion: "22085" uid: 9cb2c34f-a12a-4302-bce1-3f04677f7c5a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:01:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-ff3865-predictor-6b98f7f6fb-krwgl namespace: kserve-ci-e2e-test resourceVersion: "22027" uid: db0b0dfb-c87f-4735-b90f-558a56c7df01 kind: Event lastTimestamp: "2026-07-08T17:01:28Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T17:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:01:28Z" name: isvc-secondary-ff3865-predictor-6b98f7f6fb-krwgl.18c05f29d037fe8e namespace: kserve-ci-e2e-test resourceVersion: "22091" uid: 9efc2968-0639-4ee8-a310-47aeabba2382 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:01:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-ff3865-predictor-6b98f7f6fb-krwgl namespace: kserve-ci-e2e-test resourceVersion: "22027" uid: db0b0dfb-c87f-4735-b90f-558a56c7df01 kind: Event lastTimestamp: "2026-07-08T17:01:28Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T17:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:01:28Z" name: isvc-secondary-ff3865-predictor-6b98f7f6fb-krwgl.18c05f29d0ae04aa namespace: kserve-ci-e2e-test resourceVersion: "22092" uid: 1da9e1e8-e93f-4ffa-927a-db879c48978b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:01:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-ff3865-predictor-6b98f7f6fb-krwgl namespace: kserve-ci-e2e-test resourceVersion: "22027" uid: db0b0dfb-c87f-4735-b90f-558a56c7df01 kind: Event lastTimestamp: "2026-07-08T17:01:34Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-ff3865-predictor-6b98f7f6fb-krwgl_kserve-ci-e2e-test(db0b0dfb-c87f-4735-b90f-558a56c7df01) metadata: creationTimestamp: "2026-07-08T17:01:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:01:34Z" name: isvc-secondary-ff3865-predictor-6b98f7f6fb-krwgl.18c05f2bed8adcc0 namespace: kserve-ci-e2e-test resourceVersion: "22131" uid: e750b5e6-5dfc-4c42-9c0e-1007f82f6d69 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:01:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-ff3865-predictor-6b98f7f6fb namespace: kserve-ci-e2e-test resourceVersion: "22017" uid: 6ec86754-32f4-4991-b7b7-4951edefa671 kind: Event lastTimestamp: "2026-07-08T17:01:24Z" message: 'Created pod: isvc-secondary-ff3865-predictor-6b98f7f6fb-krwgl' metadata: creationTimestamp: "2026-07-08T17:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:01:24Z" name: isvc-secondary-ff3865-predictor-6b98f7f6fb.18c05f298e28e46f namespace: kserve-ci-e2e-test resourceVersion: "22028" uid: 7f8e7dd6-9f33-4670-a22c-c367f66817f3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:01:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-ff3865-predictor namespace: kserve-ci-e2e-test resourceVersion: "22015" uid: 49ea8a0b-bed9-4cd8-8c85-ea94a48fd425 kind: Event lastTimestamp: "2026-07-08T17:01:24Z" message: Scaled up replica set isvc-secondary-ff3865-predictor-6b98f7f6fb from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:01:24Z" name: isvc-secondary-ff3865-predictor.18c05f2988e2c25d namespace: kserve-ci-e2e-test resourceVersion: "22018" uid: 859b43d2-0ffa-44e1-8a3d-72a9ef8bda03 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:01:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-ff3865-predictor namespace: kserve-ci-e2e-test resourceVersion: "22016" uid: 34ab7174-2167-4127-9cbe-2cc99a538643 kind: Event lastTimestamp: "2026-07-08T17:01:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:01:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:01:39Z" name: isvc-secondary-ff3865-predictor.18c05f2d07735842 namespace: kserve-ci-e2e-test resourceVersion: "22169" uid: 70f92aaf-85a1-426f-957c-492207666c80 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:01:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-ff3865-predictor namespace: kserve-ci-e2e-test resourceVersion: "22016" uid: 34ab7174-2167-4127-9cbe-2cc99a538643 kind: Event lastTimestamp: "2026-07-08T17:01:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:01:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:01:39Z" name: isvc-secondary-ff3865-predictor.18c05f2d07db5fac namespace: kserve-ci-e2e-test resourceVersion: "22172" uid: 4918a4b4-5d8d-493c-ad07-e59627e9514b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-07-08T17:01:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-ff3865 namespace: kserve-ci-e2e-test resourceVersion: "22008" uid: 072d4c43-2ef0-46db-9383-3a94d67aa5ac kind: Event lastTimestamp: "2026-07-08T17:01:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T17:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:01:34Z" name: isvc-secondary-ff3865.18c05f2988ec0035 namespace: kserve-ci-e2e-test resourceVersion: "22137" uid: 87ee6f1e-3feb-45f5-a2e1-0c21efaf9911 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:01:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-ff3865 namespace: kserve-ci-e2e-test resourceVersion: "22008" uid: 072d4c43-2ef0-46db-9383-3a94d67aa5ac kind: Event lastTimestamp: "2026-07-08T17:01:24Z" message: 'Failed to update status for InferenceService "isvc-secondary-ff3865": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-ff3865": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:01:24Z" name: isvc-secondary-ff3865.18c05f2989640695 namespace: kserve-ci-e2e-test resourceVersion: "22036" uid: 4f322ba9-a32f-4cb4-9617-9382903ed77d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:01:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-ff3865 namespace: kserve-ci-e2e-test resourceVersion: "22008" uid: 072d4c43-2ef0-46db-9383-3a94d67aa5ac kind: Event lastTimestamp: "2026-07-08T17:01:24Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-ff3865": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:01:24Z" name: isvc-secondary-ff3865.18c05f2989646d1e namespace: kserve-ci-e2e-test resourceVersion: "22038" uid: 7aa58129-4fad-4988-a50e-5e62c9637804 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:51:42.882058Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs namespace: kserve-ci-e2e-test resourceVersion: "17015" uid: aa6f46ad-52fc-46d4-a305-394892cb18a1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs to ip-10-0-142-94.ec2.internal metadata: creationTimestamp: "2026-07-08T16:51:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:51:42Z" name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs.18c05ea221b6e3af namespace: kserve-ci-e2e-test resourceVersion: "17024" uid: bdb62f15-1eb3-4e7c-8b78-18bdd5bcfb50 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd6c76d55-gncps source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: aa6f46ad-52fc-46d4-a305-394892cb18a1 kind: Event lastTimestamp: "2026-07-08T16:51:43Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-raw-5d281-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:43Z" name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs.18c05ea22ee96645 namespace: kserve-ci-e2e-test resourceVersion: "17058" uid: dce70787-3d98-427b-8aa0-2f445ef08370 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs namespace: kserve-ci-e2e-test resourceVersion: "17036" uid: aa6f46ad-52fc-46d4-a305-394892cb18a1 kind: Event lastTimestamp: "2026-07-08T16:51:43Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:51:43Z" name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs.18c05ea260755653 namespace: kserve-ci-e2e-test resourceVersion: "17068" uid: 006dacf7-7b2e-4007-9172-dcb0d0c18d79 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: aa6f46ad-52fc-46d4-a305-394892cb18a1 kind: Event lastTimestamp: "2026-07-08T16:51:43Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7a9c075ae5e64d970cc026f71d8348f193fda0ed6de6807045e29c208821be1e" already present on machine metadata: creationTimestamp: "2026-07-08T16:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:43Z" name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs.18c05ea261e507a0 namespace: kserve-ci-e2e-test resourceVersion: "17070" uid: 0fb1371d-e75b-4905-ba7f-4dc60f206647 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: aa6f46ad-52fc-46d4-a305-394892cb18a1 kind: Event lastTimestamp: "2026-07-08T16:51:44Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:51:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:44Z" name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs.18c05ea2660aed0b namespace: kserve-ci-e2e-test resourceVersion: "17074" uid: 79af83c7-4968-4fff-943c-732cd2d1ff39 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: aa6f46ad-52fc-46d4-a305-394892cb18a1 kind: Event lastTimestamp: "2026-07-08T16:51:44Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:51:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:44Z" name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs.18c05ea266914782 namespace: kserve-ci-e2e-test resourceVersion: "17076" uid: 1a4bf2bf-f195-4ee5-bd5a-f70c0a41aaf2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: aa6f46ad-52fc-46d4-a305-394892cb18a1 kind: Event lastTimestamp: "2026-07-08T16:51:48Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T16:51:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:48Z" name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs.18c05ea3606892ae namespace: kserve-ci-e2e-test resourceVersion: "17121" uid: 9b2b4146-7a94-46c6-920e-11ef3e0d2ff2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: aa6f46ad-52fc-46d4-a305-394892cb18a1 kind: Event lastTimestamp: "2026-07-08T16:51:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:51:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:48Z" name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs.18c05ea363d8d9e8 namespace: kserve-ci-e2e-test resourceVersion: "17127" uid: 43baeb6f-8609-443f-94c7-7cd17ed1903b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: aa6f46ad-52fc-46d4-a305-394892cb18a1 kind: Event lastTimestamp: "2026-07-08T16:51:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:51:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:48Z" name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs.18c05ea3644ca65f namespace: kserve-ci-e2e-test resourceVersion: "17130" uid: 08be7a84-163e-4470-8ec7-ff7675f1c7ee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: aa6f46ad-52fc-46d4-a305-394892cb18a1 kind: Event lastTimestamp: "2026-07-08T16:51:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:51:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:48Z" name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs.18c05ea364590b26 namespace: kserve-ci-e2e-test resourceVersion: "17131" uid: e0404d70-a020-4f8f-af93-63ad38f61012 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: aa6f46ad-52fc-46d4-a305-394892cb18a1 kind: Event lastTimestamp: "2026-07-08T16:51:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:51:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:48Z" name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs.18c05ea36732441c namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: f7c7e420-4232-470c-9d00-2e40499c9331 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: aa6f46ad-52fc-46d4-a305-394892cb18a1 kind: Event lastTimestamp: "2026-07-08T16:51:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:51:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:48Z" name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs.18c05ea367ad036d namespace: kserve-ci-e2e-test resourceVersion: "17135" uid: 9869f490-65a4-44ff-8c8f-01bdb226dad7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T16:51:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: aa6f46ad-52fc-46d4-a305-394892cb18a1 kind: Event lastTimestamp: "2026-07-08T16:53:35Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:51:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:35Z" name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs.18c05ea39ca92014 namespace: kserve-ci-e2e-test resourceVersion: "18140" uid: 4c3c6ff2-7e20-497d-bd5d-7b394719991f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: aa6f46ad-52fc-46d4-a305-394892cb18a1 kind: Event lastTimestamp: "2026-07-08T16:53:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:33Z" name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs.18c05ebbd278c1a8 namespace: kserve-ci-e2e-test resourceVersion: "18028" uid: 49157ecb-6000-4c92-af40-5dffb0b5abc6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: aa6f46ad-52fc-46d4-a305-394892cb18a1 kind: Event lastTimestamp: "2026-07-08T16:53:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:33Z" name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs.18c05ebbd2792600 namespace: kserve-ci-e2e-test resourceVersion: "18030" uid: 0ded1a1f-30c3-401a-8356-375c088a45af reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: aa6f46ad-52fc-46d4-a305-394892cb18a1 kind: Event lastTimestamp: "2026-07-08T16:53:35Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:35Z" name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs.18c05ebc4b12f5d0 namespace: kserve-ci-e2e-test resourceVersion: "18138" uid: c971ffdb-48ae-48ad-a6e2-ee55eb04a886 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb namespace: kserve-ci-e2e-test resourceVersion: "17004" uid: 81f6cce2-3414-436c-a549-e2ca2f154ea8 kind: Event lastTimestamp: "2026-07-08T16:51:42Z" message: 'Created pod: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb-xxbfs' metadata: creationTimestamp: "2026-07-08T16:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:51:42Z" name: isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb.18c05ea22115d6a7 namespace: kserve-ci-e2e-test resourceVersion: "17018" uid: 4359ed4c-5db8-40a4-a8c6-65ac64dcfdf5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-5d281-predictor namespace: kserve-ci-e2e-test resourceVersion: "17002" uid: 0f7ce193-3cda-46be-a2de-6056085bf25a kind: Event lastTimestamp: "2026-07-08T16:51:42Z" message: Scaled up replica set isvc-sklearn-graph-raw-5d281-predictor-6d6796fdb from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:51:42Z" name: isvc-sklearn-graph-raw-5d281-predictor.18c05ea21c934caa namespace: kserve-ci-e2e-test resourceVersion: "17005" uid: 5345c0fc-2968-4cfb-98e3-9831d8f5340b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:51:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-5d281-predictor namespace: kserve-ci-e2e-test resourceVersion: "17003" uid: 45bc82d9-cbfd-4584-a20e-59e1dac432cb kind: Event lastTimestamp: "2026-07-08T16:52:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:51:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:52:27Z" name: isvc-sklearn-graph-raw-5d281-predictor.18c05ea59b610991 namespace: kserve-ci-e2e-test resourceVersion: "17453" uid: 87e82bb1-290b-49e7-8bc5-76607a2d3dce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:51:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-5d281-predictor namespace: kserve-ci-e2e-test resourceVersion: "17003" uid: 45bc82d9-cbfd-4584-a20e-59e1dac432cb kind: Event lastTimestamp: "2026-07-08T16:52:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:51:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:52:27Z" name: isvc-sklearn-graph-raw-5d281-predictor.18c05ea59bc5a1c8 namespace: kserve-ci-e2e-test resourceVersion: "17454" uid: d8e5a425-3543-4813-8f2b-2ab37ca8f4ee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:52:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-5d281-predictor namespace: kserve-ci-e2e-test resourceVersion: "17216" uid: 45bc82d9-cbfd-4584-a20e-59e1dac432cb kind: Event lastTimestamp: "2026-07-08T16:53:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:53:27Z" name: isvc-sklearn-graph-raw-5d281-predictor.18c05ea91a8d329f namespace: kserve-ci-e2e-test resourceVersion: "17935" uid: e625a0a6-185a-401b-aafc-928662ac2da7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:52:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-5d281-predictor namespace: kserve-ci-e2e-test resourceVersion: "17216" uid: 45bc82d9-cbfd-4584-a20e-59e1dac432cb kind: Event lastTimestamp: "2026-07-08T16:53:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:53:27Z" name: isvc-sklearn-graph-raw-5d281-predictor.18c05ea91afbd181 namespace: kserve-ci-e2e-test resourceVersion: "17936" uid: 3b917404-5825-4651-9e3a-76e635004414 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T16:51:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-5d281 namespace: kserve-ci-e2e-test resourceVersion: "16993" uid: 03d3e606-6936-43f4-a26b-4eda3e25b74e kind: Event lastTimestamp: "2026-07-08T16:53:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:53:05Z" name: isvc-sklearn-graph-raw-5d281.18c05ea21c9c6fdd namespace: kserve-ci-e2e-test resourceVersion: "17736" uid: 3156d0ae-ec66-49bc-afac-3b61f1a2c8d0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-5d281 namespace: kserve-ci-e2e-test resourceVersion: "17731" uid: 03d3e606-6936-43f4-a26b-4eda3e25b74e kind: Event lastTimestamp: "2026-07-08T16:53:05Z" message: InferenceService [isvc-sklearn-graph-raw-5d281] is Ready metadata: creationTimestamp: "2026-07-08T16:53:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:53:05Z" name: isvc-sklearn-graph-raw-5d281.18c05eb5578cfd43 namespace: kserve-ci-e2e-test resourceVersion: "17734" uid: 96f2525e-6064-49ad-9727-c20b6c251c7e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:53:33.318485Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb namespace: kserve-ci-e2e-test resourceVersion: "18037" uid: eb53586a-ea54-4def-87dc-9bdc6baa1761 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb to ip-10-0-142-94.ec2.internal metadata: creationTimestamp: "2026-07-08T16:53:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:53:33Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb.18c05ebbd83d20b0 namespace: kserve-ci-e2e-test resourceVersion: "18046" uid: 18d23d77-dc0b-4a43-91a7-b3353224df43 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd6c76d55-gncps source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:33Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb namespace: kserve-ci-e2e-test resourceVersion: "18055" uid: eb53586a-ea54-4def-87dc-9bdc6baa1761 kind: Event lastTimestamp: "2026-07-08T16:53:33Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:53:33Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb.18c05ebbf24ac1d7 namespace: kserve-ci-e2e-test resourceVersion: "18103" uid: 89361947-15f6-4315-ab85-952c9653174f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb namespace: kserve-ci-e2e-test resourceVersion: "18039" uid: eb53586a-ea54-4def-87dc-9bdc6baa1761 kind: Event lastTimestamp: "2026-07-08T16:53:33Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7a9c075ae5e64d970cc026f71d8348f193fda0ed6de6807045e29c208821be1e" already present on machine metadata: creationTimestamp: "2026-07-08T16:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:33Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb.18c05ebbf4067804 namespace: kserve-ci-e2e-test resourceVersion: "18106" uid: 58625bcc-b321-47a1-b9e7-9a26db62f637 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb namespace: kserve-ci-e2e-test resourceVersion: "18039" uid: eb53586a-ea54-4def-87dc-9bdc6baa1761 kind: Event lastTimestamp: "2026-07-08T16:53:33Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:33Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb.18c05ebbf6d09d1a namespace: kserve-ci-e2e-test resourceVersion: "18108" uid: a71c10d0-95f3-4410-9c05-eabbe0c0a924 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb namespace: kserve-ci-e2e-test resourceVersion: "18039" uid: eb53586a-ea54-4def-87dc-9bdc6baa1761 kind: Event lastTimestamp: "2026-07-08T16:53:33Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:33Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb.18c05ebbf734f739 namespace: kserve-ci-e2e-test resourceVersion: "18109" uid: e26295cd-6402-49d5-8db5-0bf6a61c414d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb namespace: kserve-ci-e2e-test resourceVersion: "18039" uid: eb53586a-ea54-4def-87dc-9bdc6baa1761 kind: Event lastTimestamp: "2026-07-08T16:53:37Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T16:53:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:37Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb.18c05ebcdea86ed3 namespace: kserve-ci-e2e-test resourceVersion: "18163" uid: 8d6700ea-e084-4990-b11d-e2d9b908c636 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb namespace: kserve-ci-e2e-test resourceVersion: "18039" uid: eb53586a-ea54-4def-87dc-9bdc6baa1761 kind: Event lastTimestamp: "2026-07-08T16:53:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:53:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:37Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb.18c05ebce27ec5a5 namespace: kserve-ci-e2e-test resourceVersion: "18167" uid: f833e7c8-effd-4787-b5e8-41a6730986f4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb namespace: kserve-ci-e2e-test resourceVersion: "18039" uid: eb53586a-ea54-4def-87dc-9bdc6baa1761 kind: Event lastTimestamp: "2026-07-08T16:53:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:53:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:37Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb.18c05ebce2ef0bce namespace: kserve-ci-e2e-test resourceVersion: "18168" uid: 2659aa64-2b27-4cdd-b03d-3ecefbdd0440 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb namespace: kserve-ci-e2e-test resourceVersion: "18039" uid: eb53586a-ea54-4def-87dc-9bdc6baa1761 kind: Event lastTimestamp: "2026-07-08T16:53:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:53:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:37Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb.18c05ebce2fa2a1d namespace: kserve-ci-e2e-test resourceVersion: "18169" uid: b8dc2d94-461e-4958-b3e4-bd2fdea5096f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb namespace: kserve-ci-e2e-test resourceVersion: "18039" uid: eb53586a-ea54-4def-87dc-9bdc6baa1761 kind: Event lastTimestamp: "2026-07-08T16:53:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:53:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:37Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb.18c05ebce5e3db39 namespace: kserve-ci-e2e-test resourceVersion: "18172" uid: 39bd3804-c1cc-4496-9ff1-7eaffb486bb3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb namespace: kserve-ci-e2e-test resourceVersion: "18039" uid: eb53586a-ea54-4def-87dc-9bdc6baa1761 kind: Event lastTimestamp: "2026-07-08T16:53:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:53:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:37Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb.18c05ebce6493b19 namespace: kserve-ci-e2e-test resourceVersion: "18173" uid: 736bc06d-6347-4c65-972f-0942d33f501e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T16:53:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb namespace: kserve-ci-e2e-test resourceVersion: "18039" uid: eb53586a-ea54-4def-87dc-9bdc6baa1761 kind: Event lastTimestamp: "2026-07-08T16:55:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:53:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:24Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb.18c05ebd1b17d69f namespace: kserve-ci-e2e-test resourceVersion: "19136" uid: df7e9b9a-5db8-49ed-9eee-9ebe862fdd2c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb namespace: kserve-ci-e2e-test resourceVersion: "18039" uid: eb53586a-ea54-4def-87dc-9bdc6baa1761 kind: Event lastTimestamp: "2026-07-08T16:55:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:23Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb.18c05ed5805a5848 namespace: kserve-ci-e2e-test resourceVersion: "19083" uid: d90d8976-7c7b-4265-9ad1-4aa6832fb2f9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb namespace: kserve-ci-e2e-test resourceVersion: "18039" uid: eb53586a-ea54-4def-87dc-9bdc6baa1761 kind: Event lastTimestamp: "2026-07-08T16:55:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:23Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb.18c05ed5805aeac0 namespace: kserve-ci-e2e-test resourceVersion: "19080" uid: 2ac74ab0-b3c7-4845-a02b-4f28e871e1df reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb namespace: kserve-ci-e2e-test resourceVersion: "18039" uid: eb53586a-ea54-4def-87dc-9bdc6baa1761 kind: Event lastTimestamp: "2026-07-08T16:55:24Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:24Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb.18c05ed5c98f8369 namespace: kserve-ci-e2e-test resourceVersion: "19135" uid: e437d736-e3ad-4795-9cd6-28119400a088 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878 namespace: kserve-ci-e2e-test resourceVersion: "18012" uid: e37e93fa-c961-4168-ac3e-241328ad563f kind: Event lastTimestamp: "2026-07-08T16:53:33Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878-5v2kb' metadata: creationTimestamp: "2026-07-08T16:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:53:33Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878.18c05ebbd74e7a5d namespace: kserve-ci-e2e-test resourceVersion: "18041" uid: f91b472c-4f2f-4dc6-a664-5c6b6c199e47 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-3e871-predictor namespace: kserve-ci-e2e-test resourceVersion: "18006" uid: 6f2cae97-4ad6-4e6b-993b-5d7ea43cb0f5 kind: Event lastTimestamp: "2026-07-08T16:53:33Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-3e871-predictor-78f774c878 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:53:33Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor.18c05ebbd086a12a namespace: kserve-ci-e2e-test resourceVersion: "18019" uid: 26f9da46-b988-40d4-8ec4-25dfd0d7c1bc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:53:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3e871-predictor namespace: kserve-ci-e2e-test resourceVersion: "18010" uid: 4c08e725-1d48-4e4e-8079-bc757d0c4535 kind: Event lastTimestamp: "2026-07-08T16:54:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:03Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor.18c05ebf4eaec0a3 namespace: kserve-ci-e2e-test resourceVersion: "18389" uid: 07c19035-624e-477b-9af6-6601ec83a6a3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:53:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3e871-predictor namespace: kserve-ci-e2e-test resourceVersion: "18010" uid: 4c08e725-1d48-4e4e-8079-bc757d0c4535 kind: Event lastTimestamp: "2026-07-08T16:54:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:03Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor.18c05ebf4f164c4d namespace: kserve-ci-e2e-test resourceVersion: "18390" uid: 1bcc229e-0baf-4976-ba7a-2f74e2fa3cca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:54:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3e871-predictor namespace: kserve-ci-e2e-test resourceVersion: "18269" uid: 4c08e725-1d48-4e4e-8079-bc757d0c4535 kind: Event lastTimestamp: "2026-07-08T16:55:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:18Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor.18c05ec64d715c9d namespace: kserve-ci-e2e-test resourceVersion: "18982" uid: e8161d6a-b35b-4a42-aeb3-4dc465fb507f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-08T16:54:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-3e871-predictor namespace: kserve-ci-e2e-test resourceVersion: "18269" uid: 4c08e725-1d48-4e4e-8079-bc757d0c4535 kind: Event lastTimestamp: "2026-07-08T16:55:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:18Z" name: isvc-sklearn-graph-raw-hpa-3e871-predictor.18c05ec64dd80565 namespace: kserve-ci-e2e-test resourceVersion: "18983" uid: 583648c8-788e-41ac-98d9-6a1843a766f8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-08T16:53:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-3e871 namespace: kserve-ci-e2e-test resourceVersion: "17999" uid: 02b48227-158f-412e-b4f8-c4da90aac716 kind: Event lastTimestamp: "2026-07-08T16:54:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:54:54Z" name: isvc-sklearn-graph-raw-hpa-3e871.18c05ebbd0723c10 namespace: kserve-ci-e2e-test resourceVersion: "18787" uid: cfe2d555-a5f5-44bf-afd4-0b2d0a8b9ebd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-3e871 namespace: kserve-ci-e2e-test resourceVersion: "17999" uid: 02b48227-158f-412e-b4f8-c4da90aac716 kind: Event lastTimestamp: "2026-07-08T16:53:33Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-3e871": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-3e871": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:53:33Z" name: isvc-sklearn-graph-raw-hpa-3e871.18c05ebbd1c28444 namespace: kserve-ci-e2e-test resourceVersion: "18026" uid: 85aa6ba6-3a6a-45f9-aed2-52abbf356dfe reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-3e871 namespace: kserve-ci-e2e-test resourceVersion: "18785" uid: 02b48227-158f-412e-b4f8-c4da90aac716 kind: Event lastTimestamp: "2026-07-08T16:54:54Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-3e871] is Ready metadata: creationTimestamp: "2026-07-08T16:54:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:54:54Z" name: isvc-sklearn-graph-raw-hpa-3e871.18c05eced3bee729 namespace: kserve-ci-e2e-test resourceVersion: "18786" uid: f5218496-2bae-4932-aa8f-c974a3220f34 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:57:08.970679Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr namespace: kserve-ci-e2e-test resourceVersion: "20034" uid: b3b76551-8e16-4370-a7f3-a5a91577cd3b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr to ip-10-0-142-94.ec2.internal metadata: creationTimestamp: "2026-07-08T16:57:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:57:08Z" name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr.18c05eee0e1c5e4c namespace: kserve-ci-e2e-test resourceVersion: "20041" uid: a75382dd-8b6b-4669-82d8-5210d5d3d519 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd6c76d55-gncps source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:09Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: b3b76551-8e16-4370-a7f3-a5a91577cd3b kind: Event lastTimestamp: "2026-07-08T16:57:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-c7447-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:57:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:09Z" name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr.18c05eee160f8848 namespace: kserve-ci-e2e-test resourceVersion: "20063" uid: 12a3067b-0f5b-4fd6-9e32-27d256631565 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr namespace: kserve-ci-e2e-test resourceVersion: "20055" uid: b3b76551-8e16-4370-a7f3-a5a91577cd3b kind: Event lastTimestamp: "2026-07-08T16:57:10Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:57:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:57:10Z" name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr.18c05eee4d56c518 namespace: kserve-ci-e2e-test resourceVersion: "20076" uid: d615e9c2-9c4d-4fd0-9169-58c44ef6e7f9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: b3b76551-8e16-4370-a7f3-a5a91577cd3b kind: Event lastTimestamp: "2026-07-08T16:57:10Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7a9c075ae5e64d970cc026f71d8348f193fda0ed6de6807045e29c208821be1e" already present on machine metadata: creationTimestamp: "2026-07-08T16:57:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:10Z" name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr.18c05eee4ee091ee namespace: kserve-ci-e2e-test resourceVersion: "20078" uid: 25ab17f9-9df3-4076-b711-0bbddaae068d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: b3b76551-8e16-4370-a7f3-a5a91577cd3b kind: Event lastTimestamp: "2026-07-08T16:57:10Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:57:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:10Z" name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr.18c05eee51f35946 namespace: kserve-ci-e2e-test resourceVersion: "20079" uid: 6890b66c-e6d7-4e59-96df-ab70f6e2f4b0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: b3b76551-8e16-4370-a7f3-a5a91577cd3b kind: Event lastTimestamp: "2026-07-08T16:57:10Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:57:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:10Z" name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr.18c05eee529abe50 namespace: kserve-ci-e2e-test resourceVersion: "20080" uid: 46ecc8ac-3be7-4895-8033-6b6f191147fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: b3b76551-8e16-4370-a7f3-a5a91577cd3b kind: Event lastTimestamp: "2026-07-08T16:57:14Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T16:57:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:14Z" name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr.18c05eef60afd581 namespace: kserve-ci-e2e-test resourceVersion: "20116" uid: 05dc63b3-e4eb-40e6-ab77-030d92eb3b57 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: b3b76551-8e16-4370-a7f3-a5a91577cd3b kind: Event lastTimestamp: "2026-07-08T16:57:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:57:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:14Z" name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr.18c05eef63e29ce3 namespace: kserve-ci-e2e-test resourceVersion: "20119" uid: 06cb674a-b49d-4fc0-a79b-0794ff761580 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: b3b76551-8e16-4370-a7f3-a5a91577cd3b kind: Event lastTimestamp: "2026-07-08T16:57:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:57:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:14Z" name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr.18c05eef64632b11 namespace: kserve-ci-e2e-test resourceVersion: "20120" uid: bf1127a3-07b0-46d2-b6d8-e76c7e5122cb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: b3b76551-8e16-4370-a7f3-a5a91577cd3b kind: Event lastTimestamp: "2026-07-08T16:57:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:57:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:14Z" name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr.18c05eef646e6989 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: db311d58-f0ad-463c-bf94-bf693eefa7b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: b3b76551-8e16-4370-a7f3-a5a91577cd3b kind: Event lastTimestamp: "2026-07-08T16:57:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:57:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:14Z" name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr.18c05eef67bfa24d namespace: kserve-ci-e2e-test resourceVersion: "20124" uid: 9b0db29a-a84e-4953-942d-d39557705bbb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: b3b76551-8e16-4370-a7f3-a5a91577cd3b kind: Event lastTimestamp: "2026-07-08T16:57:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:57:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:14Z" name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr.18c05eef683f41f2 namespace: kserve-ci-e2e-test resourceVersion: "20126" uid: 24786ab8-8de7-4851-8951-b736bed12215 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 18 eventTime: null firstTimestamp: "2026-07-08T16:57:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: b3b76551-8e16-4370-a7f3-a5a91577cd3b kind: Event lastTimestamp: "2026-07-08T16:59:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:57:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:58Z" name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr.18c05eefd8c7a454 namespace: kserve-ci-e2e-test resourceVersion: "21366" uid: 901186b8-690b-4ffe-b245-c2c76c5bc5d7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: b3b76551-8e16-4370-a7f3-a5a91577cd3b kind: Event lastTimestamp: "2026-07-08T16:59:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:59:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:49Z" name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr.18c05f136076990f namespace: kserve-ci-e2e-test resourceVersion: "21252" uid: 9a5b4951-95a2-4873-b344-64c4e020cbdc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: b3b76551-8e16-4370-a7f3-a5a91577cd3b kind: Event lastTimestamp: "2026-07-08T16:59:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:59:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:49Z" name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr.18c05f136077218b namespace: kserve-ci-e2e-test resourceVersion: "21253" uid: 89c0487a-aa09-4361-a00b-d627568fa444 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:59:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: b3b76551-8e16-4370-a7f3-a5a91577cd3b kind: Event lastTimestamp: "2026-07-08T16:59:57Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:59:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:59:57Z" name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr.18c05f142b4f6be2 namespace: kserve-ci-e2e-test resourceVersion: "21358" uid: 2e0c6a96-7e6e-4a76-9593-551e3fa20d8b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6 namespace: kserve-ci-e2e-test resourceVersion: "20018" uid: df2701fa-846a-49fd-b6c4-180d929fd06f kind: Event lastTimestamp: "2026-07-08T16:57:08Z" message: 'Created pod: isvc-sklearn-scale-raw-c7447-predictor-86797847c6-zdmwr' metadata: creationTimestamp: "2026-07-08T16:57:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:57:08Z" name: isvc-sklearn-scale-raw-c7447-predictor-86797847c6.18c05eee0d639adc namespace: kserve-ci-e2e-test resourceVersion: "20036" uid: 37635412-6f6e-4dca-84ce-533d3368100a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-c7447-predictor namespace: kserve-ci-e2e-test resourceVersion: "20013" uid: e8205608-2a9d-4939-80e8-5e3a4eb9c938 kind: Event lastTimestamp: "2026-07-08T16:57:08Z" message: Scaled up replica set isvc-sklearn-scale-raw-c7447-predictor-86797847c6 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:57:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:57:08Z" name: isvc-sklearn-scale-raw-c7447-predictor.18c05eee080b0c24 namespace: kserve-ci-e2e-test resourceVersion: "20021" uid: e0863790-1d4d-492e-a478-dd1ee5301c3d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:57:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-c7447-predictor namespace: kserve-ci-e2e-test resourceVersion: "20017" uid: 0884e00c-f421-4a8f-ab66-7da534d0e119 kind: Event lastTimestamp: "2026-07-08T16:57:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:57:53Z" name: isvc-sklearn-scale-raw-c7447-predictor.18c05ef187043324 namespace: kserve-ci-e2e-test resourceVersion: "20417" uid: 4a945778-a225-4ca3-8550-f1878b0b39a5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:57:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-c7447-predictor namespace: kserve-ci-e2e-test resourceVersion: "20017" uid: 0884e00c-f421-4a8f-ab66-7da534d0e119 kind: Event lastTimestamp: "2026-07-08T16:57:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:57:53Z" name: isvc-sklearn-scale-raw-c7447-predictor.18c05ef1875d0f24 namespace: kserve-ci-e2e-test resourceVersion: "20419" uid: 2961c363-4061-410f-9f55-0def6daf64a0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-08T16:57:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-c7447-predictor namespace: kserve-ci-e2e-test resourceVersion: "20198" uid: 0884e00c-f421-4a8f-ab66-7da534d0e119 kind: Event lastTimestamp: "2026-07-08T16:59:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:39Z" name: isvc-sklearn-scale-raw-c7447-predictor.18c05ef506463285 namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: f9198b53-7dc9-4b5c-9e73-ce20e0951191 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-08T16:57:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-c7447-predictor namespace: kserve-ci-e2e-test resourceVersion: "20198" uid: 0884e00c-f421-4a8f-ab66-7da534d0e119 kind: Event lastTimestamp: "2026-07-08T16:59:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:59:39Z" name: isvc-sklearn-scale-raw-c7447-predictor.18c05ef506ad28a6 namespace: kserve-ci-e2e-test resourceVersion: "21153" uid: 3e25bf8b-8ae8-46d2-88a9-0d73453923a5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-08T16:57:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-c7447 namespace: kserve-ci-e2e-test resourceVersion: "19994" uid: 4abc5bef-6b41-4295-9e1c-fc00b5a3e05e kind: Event lastTimestamp: "2026-07-08T16:59:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:57:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:59:48Z" name: isvc-sklearn-scale-raw-c7447.18c05eee085a6cc7 namespace: kserve-ci-e2e-test resourceVersion: "21231" uid: 83ddbb3c-497d-4079-aea3-24c683e19fde reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-c7447 namespace: kserve-ci-e2e-test resourceVersion: "19994" uid: 4abc5bef-6b41-4295-9e1c-fc00b5a3e05e kind: Event lastTimestamp: "2026-07-08T16:57:08Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-c7447": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-c7447": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:57:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:57:08Z" name: isvc-sklearn-scale-raw-c7447.18c05eee091f048f namespace: kserve-ci-e2e-test resourceVersion: "20024" uid: 095ef06e-20ae-4f9e-b852-1f0f0006c2ed reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:59:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-c7447 namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: 4abc5bef-6b41-4295-9e1c-fc00b5a3e05e kind: Event lastTimestamp: "2026-07-08T16:59:48Z" message: InferenceService [isvc-sklearn-scale-raw-c7447] is Ready metadata: creationTimestamp: "2026-07-08T16:59:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:59:48Z" name: isvc-sklearn-scale-raw-c7447.18c05f132cc86a58 namespace: kserve-ci-e2e-test resourceVersion: "21230" uid: 6af701f9-88ec-4925-9399-933df3593d2e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:51:42.953675Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz namespace: kserve-ci-e2e-test resourceVersion: "17039" uid: 435847a0-f882-4d91-9d21-e96f272fe5fb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz to ip-10-0-142-94.ec2.internal metadata: creationTimestamp: "2026-07-08T16:51:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:51:42Z" name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz.18c05ea225fbbf4d namespace: kserve-ci-e2e-test resourceVersion: "17044" uid: 961d61c3-4d60-44f1-9cf4-0ac609f55d8f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd6c76d55-gncps source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: 435847a0-f882-4d91-9d21-e96f272fe5fb kind: Event lastTimestamp: "2026-07-08T16:51:43Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-5d281-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:43Z" name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz.18c05ea22ee47d6f namespace: kserve-ci-e2e-test resourceVersion: "17057" uid: 816b83ae-dd94-45b5-981d-7875b13ffeb5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:44Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz namespace: kserve-ci-e2e-test resourceVersion: "17050" uid: 435847a0-f882-4d91-9d21-e96f272fe5fb kind: Event lastTimestamp: "2026-07-08T16:51:44Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:51:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:51:44Z" name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz.18c05ea265107daa namespace: kserve-ci-e2e-test resourceVersion: "17071" uid: 9ead1f44-2e0d-455f-8f83-d462084766c1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: 435847a0-f882-4d91-9d21-e96f272fe5fb kind: Event lastTimestamp: "2026-07-08T16:51:44Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7a9c075ae5e64d970cc026f71d8348f193fda0ed6de6807045e29c208821be1e" already present on machine metadata: creationTimestamp: "2026-07-08T16:51:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:44Z" name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz.18c05ea2668f4869 namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: e59d4e93-bcc5-4903-a65d-90cdb495f536 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: 435847a0-f882-4d91-9d21-e96f272fe5fb kind: Event lastTimestamp: "2026-07-08T16:51:44Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:51:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:44Z" name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz.18c05ea26970d267 namespace: kserve-ci-e2e-test resourceVersion: "17077" uid: 007e0e77-865b-407f-8802-05b65e36b1b3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: 435847a0-f882-4d91-9d21-e96f272fe5fb kind: Event lastTimestamp: "2026-07-08T16:51:44Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:51:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:44Z" name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz.18c05ea269e2812d namespace: kserve-ci-e2e-test resourceVersion: "17078" uid: e402866e-8427-4bcd-ba8a-957b75c3420d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: 435847a0-f882-4d91-9d21-e96f272fe5fb kind: Event lastTimestamp: "2026-07-08T16:51:48Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-08T16:51:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:51:48Z" name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz.18c05ea36059185e namespace: kserve-ci-e2e-test resourceVersion: "17120" uid: 24ec8cc0-b07b-4826-a45b-80b4401cc82c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: 435847a0-f882-4d91-9d21-e96f272fe5fb kind: Event lastTimestamp: "2026-07-08T16:52:07Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 19.006s (19.006s including waiting). Image size: 1305547515 bytes.' metadata: creationTimestamp: "2026-07-08T16:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:52:07Z" name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz.18c05ea7cd40fd40 namespace: kserve-ci-e2e-test resourceVersion: "17288" uid: 29c38b26-f7ab-4c02-8280-e80c48831a79 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: 435847a0-f882-4d91-9d21-e96f272fe5fb kind: Event lastTimestamp: "2026-07-08T16:52:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:52:07Z" name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz.18c05ea7d0db3000 namespace: kserve-ci-e2e-test resourceVersion: "17289" uid: 8cffdc0f-f1fa-4a82-8242-b699d7b06135 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: 435847a0-f882-4d91-9d21-e96f272fe5fb kind: Event lastTimestamp: "2026-07-08T16:52:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:52:07Z" name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz.18c05ea7d14c0772 namespace: kserve-ci-e2e-test resourceVersion: "17290" uid: f0a6317a-f4c8-45f7-84ff-83a9a42b7340 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: 435847a0-f882-4d91-9d21-e96f272fe5fb kind: Event lastTimestamp: "2026-07-08T16:52:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:52:07Z" name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz.18c05ea7d1575035 namespace: kserve-ci-e2e-test resourceVersion: "17291" uid: d1cc2802-bd4f-4931-96da-9c1498a6b7a3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: 435847a0-f882-4d91-9d21-e96f272fe5fb kind: Event lastTimestamp: "2026-07-08T16:52:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:52:07Z" name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz.18c05ea7d71421c3 namespace: kserve-ci-e2e-test resourceVersion: "17292" uid: 62656a60-e837-4546-a19a-a8958793c2f7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: 435847a0-f882-4d91-9d21-e96f272fe5fb kind: Event lastTimestamp: "2026-07-08T16:52:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:52:07Z" name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz.18c05ea7d7892e52 namespace: kserve-ci-e2e-test resourceVersion: "17293" uid: 080e29de-105a-485a-acbb-1cf3abbf8639 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-08T16:52:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: 435847a0-f882-4d91-9d21-e96f272fe5fb kind: Event lastTimestamp: "2026-07-08T16:53:35Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:52:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:35Z" name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz.18c05ea84a9d7dba namespace: kserve-ci-e2e-test resourceVersion: "18142" uid: e96d1f44-bf42-418b-ba79-511c3cd59853 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: 435847a0-f882-4d91-9d21-e96f272fe5fb kind: Event lastTimestamp: "2026-07-08T16:53:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:33Z" name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz.18c05ebbe21c4fa9 namespace: kserve-ci-e2e-test resourceVersion: "18091" uid: 6ed0952e-12cd-44ad-b88e-b9ae4cecdd07 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: 435847a0-f882-4d91-9d21-e96f272fe5fb kind: Event lastTimestamp: "2026-07-08T16:53:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:33Z" name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz.18c05ebbe21d9971 namespace: kserve-ci-e2e-test resourceVersion: "18093" uid: 721736e8-a10f-44ba-94c9-924e9490deaf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: 435847a0-f882-4d91-9d21-e96f272fe5fb kind: Event lastTimestamp: "2026-07-08T16:53:35Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:35Z" name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz.18c05ebc50d1f52f namespace: kserve-ci-e2e-test resourceVersion: "18141" uid: e1cac5d4-b7f6-45f6-9dc7-80c3114025d6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5 namespace: kserve-ci-e2e-test resourceVersion: "17026" uid: 7ea5e6d6-f58c-46cb-88a0-d7b23a6a2402 kind: Event lastTimestamp: "2026-07-08T16:51:42Z" message: 'Created pod: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5-pmrpz' metadata: creationTimestamp: "2026-07-08T16:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:51:42Z" name: isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5.18c05ea2258646ed namespace: kserve-ci-e2e-test resourceVersion: "17040" uid: 9690bdc8-5dd6-44e4-b36c-dc2d4cf2b7f2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:51:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-5d281-predictor namespace: kserve-ci-e2e-test resourceVersion: "17023" uid: df943621-fb28-4549-8b63-1d04da78cb4b kind: Event lastTimestamp: "2026-07-08T16:51:42Z" message: Scaled up replica set isvc-xgboost-graph-raw-5d281-predictor-5b6dbc44d5 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:51:42Z" name: isvc-xgboost-graph-raw-5d281-predictor.18c05ea2229dc116 namespace: kserve-ci-e2e-test resourceVersion: "17032" uid: b9fb37e4-fba0-4556-8801-f543971ef88a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:51:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-5d281-predictor namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 74936c0f-bfa2-4bfd-873e-3fa89b704240 kind: Event lastTimestamp: "2026-07-08T16:52:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:51:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:52:42Z" name: isvc-xgboost-graph-raw-5d281-predictor.18c05ea5a1089dad namespace: kserve-ci-e2e-test resourceVersion: "17562" uid: e403d5c9-12b9-4629-92e4-ea3a719b5ae6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:51:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-5d281-predictor namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 74936c0f-bfa2-4bfd-873e-3fa89b704240 kind: Event lastTimestamp: "2026-07-08T16:52:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:51:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:52:42Z" name: isvc-xgboost-graph-raw-5d281-predictor.18c05ea5a16ea779 namespace: kserve-ci-e2e-test resourceVersion: "17563" uid: ae70064d-ddd1-4a81-b5bc-14648740dfc7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:52:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-5d281-predictor namespace: kserve-ci-e2e-test resourceVersion: "17220" uid: 74936c0f-bfa2-4bfd-873e-3fa89b704240 kind: Event lastTimestamp: "2026-07-08T16:53:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:53:27Z" name: isvc-xgboost-graph-raw-5d281-predictor.18c05eb39c9e093b namespace: kserve-ci-e2e-test resourceVersion: "17937" uid: f16e24c4-8353-49d7-89b0-8b041216971f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T16:52:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-5d281-predictor namespace: kserve-ci-e2e-test resourceVersion: "17220" uid: 74936c0f-bfa2-4bfd-873e-3fa89b704240 kind: Event lastTimestamp: "2026-07-08T16:53:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:53:28Z" name: isvc-xgboost-graph-raw-5d281-predictor.18c05eb39d10ed46 namespace: kserve-ci-e2e-test resourceVersion: "17938" uid: 3cdb9aaa-e6b3-44c9-a0df-d163f27e1a12 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T16:51:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-5d281 namespace: kserve-ci-e2e-test resourceVersion: "17010" uid: a7b66242-c004-4d6b-bcf1-a0e54b9cee82 kind: Event lastTimestamp: "2026-07-08T16:53:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:53:05Z" name: isvc-xgboost-graph-raw-5d281.18c05ea2229e15e8 namespace: kserve-ci-e2e-test resourceVersion: "17741" uid: e61ba66f-f4a2-48c0-b0f3-541da0f1164f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-5d281 namespace: kserve-ci-e2e-test resourceVersion: "17737" uid: a7b66242-c004-4d6b-bcf1-a0e54b9cee82 kind: Event lastTimestamp: "2026-07-08T16:53:05Z" message: InferenceService [isvc-xgboost-graph-raw-5d281] is Ready metadata: creationTimestamp: "2026-07-08T16:53:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:53:05Z" name: isvc-xgboost-graph-raw-5d281.18c05eb56094640f namespace: kserve-ci-e2e-test resourceVersion: "17739" uid: 1d83501b-46ab-4a95-97e7-3e9aa2296539 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:53:33.431272Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd namespace: kserve-ci-e2e-test resourceVersion: "18063" uid: 81937630-42ac-4e5d-8427-14687fe57624 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd to ip-10-0-142-94.ec2.internal metadata: creationTimestamp: "2026-07-08T16:53:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:53:33Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd.18c05ebbdef6107d namespace: kserve-ci-e2e-test resourceVersion: "18070" uid: 07be4f7d-4fb9-4583-b1b6-4386a8544135 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd6c76d55-gncps source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:33Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd namespace: kserve-ci-e2e-test resourceVersion: "18064" uid: 81937630-42ac-4e5d-8427-14687fe57624 kind: Event lastTimestamp: "2026-07-08T16:53:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-hpa-3e871-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:33Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd.18c05ebbe7746a62 namespace: kserve-ci-e2e-test resourceVersion: "18100" uid: 39cfbfaa-8e41-4f9e-ab63-9fac2651a1a3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:34Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd namespace: kserve-ci-e2e-test resourceVersion: "18089" uid: 81937630-42ac-4e5d-8427-14687fe57624 kind: Event lastTimestamp: "2026-07-08T16:53:34Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:53:34Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd.18c05ebc1c57e818 namespace: kserve-ci-e2e-test resourceVersion: "18115" uid: 7f87172c-7a94-4a3f-b30a-ae249927d648 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd namespace: kserve-ci-e2e-test resourceVersion: "18064" uid: 81937630-42ac-4e5d-8427-14687fe57624 kind: Event lastTimestamp: "2026-07-08T16:53:34Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7a9c075ae5e64d970cc026f71d8348f193fda0ed6de6807045e29c208821be1e" already present on machine metadata: creationTimestamp: "2026-07-08T16:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:34Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd.18c05ebc1de34ff7 namespace: kserve-ci-e2e-test resourceVersion: "18117" uid: 8e197390-934f-4ae0-885c-a10dac51ea42 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd namespace: kserve-ci-e2e-test resourceVersion: "18064" uid: 81937630-42ac-4e5d-8427-14687fe57624 kind: Event lastTimestamp: "2026-07-08T16:53:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T16:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:34Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd.18c05ebc20f001c5 namespace: kserve-ci-e2e-test resourceVersion: "18118" uid: 5b6be45e-d4b7-4b4a-9a27-65212bbf171c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd namespace: kserve-ci-e2e-test resourceVersion: "18064" uid: 81937630-42ac-4e5d-8427-14687fe57624 kind: Event lastTimestamp: "2026-07-08T16:53:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T16:53:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:34Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd.18c05ebc21550405 namespace: kserve-ci-e2e-test resourceVersion: "18119" uid: 6f28bf98-fe44-4a4f-9f4b-210cf111d7c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd namespace: kserve-ci-e2e-test resourceVersion: "18064" uid: 81937630-42ac-4e5d-8427-14687fe57624 kind: Event lastTimestamp: "2026-07-08T16:53:38Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-07-08T16:53:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:38Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd.18c05ebd1aba2f41 namespace: kserve-ci-e2e-test resourceVersion: "18184" uid: 94c48708-ce20-4e59-aad1-3e3e5174ba3f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd namespace: kserve-ci-e2e-test resourceVersion: "18064" uid: 81937630-42ac-4e5d-8427-14687fe57624 kind: Event lastTimestamp: "2026-07-08T16:53:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:53:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:38Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd.18c05ebd1de781a2 namespace: kserve-ci-e2e-test resourceVersion: "18189" uid: af7ab2ff-e322-4665-958f-0aab8efe47f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd namespace: kserve-ci-e2e-test resourceVersion: "18064" uid: 81937630-42ac-4e5d-8427-14687fe57624 kind: Event lastTimestamp: "2026-07-08T16:53:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:53:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:38Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd.18c05ebd1e488ff3 namespace: kserve-ci-e2e-test resourceVersion: "18190" uid: 99dfe452-a668-4a85-9440-a13e0aa8316e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd namespace: kserve-ci-e2e-test resourceVersion: "18064" uid: 81937630-42ac-4e5d-8427-14687fe57624 kind: Event lastTimestamp: "2026-07-08T16:53:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:53:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:38Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd.18c05ebd1e511e3e namespace: kserve-ci-e2e-test resourceVersion: "18192" uid: 63f96665-9556-4003-93b6-afef20d252ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd namespace: kserve-ci-e2e-test resourceVersion: "18064" uid: 81937630-42ac-4e5d-8427-14687fe57624 kind: Event lastTimestamp: "2026-07-08T16:53:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:53:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:38Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd.18c05ebd20eb681c namespace: kserve-ci-e2e-test resourceVersion: "18195" uid: 9c1d27f7-de7b-419e-86b6-4ee444859f13 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd namespace: kserve-ci-e2e-test resourceVersion: "18064" uid: 81937630-42ac-4e5d-8427-14687fe57624 kind: Event lastTimestamp: "2026-07-08T16:53:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:53:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:38Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd.18c05ebd2151255b namespace: kserve-ci-e2e-test resourceVersion: "18196" uid: 6fdb92f0-0572-407b-90d0-836cefb6d621 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-08T16:53:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd namespace: kserve-ci-e2e-test resourceVersion: "18064" uid: 81937630-42ac-4e5d-8427-14687fe57624 kind: Event lastTimestamp: "2026-07-08T16:55:25Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:53:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:25Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd.18c05ebd571d1afb namespace: kserve-ci-e2e-test resourceVersion: "19145" uid: 5181f3f0-0760-4ef1-95e6-311af5a2f9d8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd namespace: kserve-ci-e2e-test resourceVersion: "18064" uid: 81937630-42ac-4e5d-8427-14687fe57624 kind: Event lastTimestamp: "2026-07-08T16:55:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:23Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd.18c05ed58745de6e namespace: kserve-ci-e2e-test resourceVersion: "19116" uid: bf88703d-80ba-4438-bda7-095122953049 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd namespace: kserve-ci-e2e-test resourceVersion: "18064" uid: 81937630-42ac-4e5d-8427-14687fe57624 kind: Event lastTimestamp: "2026-07-08T16:55:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:23Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd.18c05ed587463823 namespace: kserve-ci-e2e-test resourceVersion: "19117" uid: d0cb1e9c-4c0a-4281-9164-39cd9679fa9f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd namespace: kserve-ci-e2e-test resourceVersion: "18064" uid: 81937630-42ac-4e5d-8427-14687fe57624 kind: Event lastTimestamp: "2026-07-08T16:55:25Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T16:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:25Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd.18c05ed60562ab31 namespace: kserve-ci-e2e-test resourceVersion: "19144" uid: 2e798777-784d-4f1c-aa46-e2d5858cabec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9 namespace: kserve-ci-e2e-test resourceVersion: "18054" uid: 466865ae-e7b1-47e6-8494-4fd291130c89 kind: Event lastTimestamp: "2026-07-08T16:53:33Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9-h6bkd' metadata: creationTimestamp: "2026-07-08T16:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:53:33Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9.18c05ebbde511a8d namespace: kserve-ci-e2e-test resourceVersion: "18065" uid: 7af40139-5bcf-4df6-bacf-56f6d89e1f1b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-3e871-predictor namespace: kserve-ci-e2e-test resourceVersion: "18052" uid: 4fab95ac-1ead-45a0-a267-9fb25b45ff23 kind: Event lastTimestamp: "2026-07-08T16:53:33Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-3e871-predictor-f9756c6d9 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:53:33Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor.18c05ebbdaa6e21a namespace: kserve-ci-e2e-test resourceVersion: "18057" uid: bc848764-88fc-48b6-96cc-c4ffb200f679 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:53:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3e871-predictor namespace: kserve-ci-e2e-test resourceVersion: "18053" uid: 14129433-b4ba-4d19-8893-16070eb83fd2 kind: Event lastTimestamp: "2026-07-08T16:54:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:03Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor.18c05ebf59281ab3 namespace: kserve-ci-e2e-test resourceVersion: "18393" uid: b33aa355-3e12-40aa-947f-941c0d25346f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:53:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3e871-predictor namespace: kserve-ci-e2e-test resourceVersion: "18053" uid: 14129433-b4ba-4d19-8893-16070eb83fd2 kind: Event lastTimestamp: "2026-07-08T16:54:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:54:03Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor.18c05ebf5998b947 namespace: kserve-ci-e2e-test resourceVersion: "18394" uid: 7d853551-1912-4280-8ce8-bc8f6e410b7e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:54:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3e871-predictor namespace: kserve-ci-e2e-test resourceVersion: "18275" uid: 14129433-b4ba-4d19-8893-16070eb83fd2 kind: Event lastTimestamp: "2026-07-08T16:55:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:03Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor.18c05ec6574a24d7 namespace: kserve-ci-e2e-test resourceVersion: "18849" uid: a61acec4-0d0e-4ae5-8471-f38d534c1167 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T16:54:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-3e871-predictor namespace: kserve-ci-e2e-test resourceVersion: "18275" uid: 14129433-b4ba-4d19-8893-16070eb83fd2 kind: Event lastTimestamp: "2026-07-08T16:55:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T16:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:03Z" name: isvc-xgboost-graph-raw-hpa-3e871-predictor.18c05ec657bb0226 namespace: kserve-ci-e2e-test resourceVersion: "18850" uid: 278acd0c-10b9-4e7c-be35-2c99bcc0f9da reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T16:53:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-3e871 namespace: kserve-ci-e2e-test resourceVersion: "18035" uid: 4ed6de9d-5294-4726-bad5-f15d2dea6a74 kind: Event lastTimestamp: "2026-07-08T16:54:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:54:45Z" name: isvc-xgboost-graph-raw-hpa-3e871.18c05ebbdaafc2c7 namespace: kserve-ci-e2e-test resourceVersion: "18710" uid: 89b07c95-ecca-4076-8835-42e7d1e6a3d6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-3e871 namespace: kserve-ci-e2e-test resourceVersion: "18035" uid: 4ed6de9d-5294-4726-bad5-f15d2dea6a74 kind: Event lastTimestamp: "2026-07-08T16:53:33Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-hpa-3e871": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-hpa-3e871": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:53:33Z" name: isvc-xgboost-graph-raw-hpa-3e871.18c05ebbdb969f32 namespace: kserve-ci-e2e-test resourceVersion: "18060" uid: c578b0b1-3ba0-4347-9878-64e4f0489d1e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:54:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-3e871 namespace: kserve-ci-e2e-test resourceVersion: "18707" uid: 4ed6de9d-5294-4726-bad5-f15d2dea6a74 kind: Event lastTimestamp: "2026-07-08T16:54:45Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-3e871] is Ready metadata: creationTimestamp: "2026-07-08T16:54:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:54:45Z" name: isvc-xgboost-graph-raw-hpa-3e871.18c05eccbb0ac7a4 namespace: kserve-ci-e2e-test resourceVersion: "18708" uid: 3f315baf-fc9a-46cf-a25a-e59cc06b946b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:55:23.564182Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn namespace: kserve-ci-e2e-test resourceVersion: "19088" uid: 4c6d57fc-3f16-4060-8fff-5279965ae841 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-9f040-predictor-5458f85b97-zlgtn to ip-10-0-142-94.ec2.internal metadata: creationTimestamp: "2026-07-08T16:55:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:55:23Z" name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn.18c05ed58364db3f namespace: kserve-ci-e2e-test resourceVersion: "19092" uid: c2a79ae3-c728-4755-a4c7-319d025ec687 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd6c76d55-gncps source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:23Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn namespace: kserve-ci-e2e-test resourceVersion: "19090" uid: 4c6d57fc-3f16-4060-8fff-5279965ae841 kind: Event lastTimestamp: "2026-07-08T16:55:23Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "message-dumper-raw-9f040-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:23Z" name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn.18c05ed58c6a3572 namespace: kserve-ci-e2e-test resourceVersion: "19121" uid: a34bcbd7-38dd-4da0-8b07-e950c8c93f64 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:24Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn namespace: kserve-ci-e2e-test resourceVersion: "19099" uid: 4c6d57fc-3f16-4060-8fff-5279965ae841 kind: Event lastTimestamp: "2026-07-08T16:55:24Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:55:24Z" name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn.18c05ed5c0e270d0 namespace: kserve-ci-e2e-test resourceVersion: "19131" uid: 666b7394-3291-46fb-819c-35c1cd08d673 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn namespace: kserve-ci-e2e-test resourceVersion: "19090" uid: 4c6d57fc-3f16-4060-8fff-5279965ae841 kind: Event lastTimestamp: "2026-07-08T16:55:24Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-08T16:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:24Z" name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn.18c05ed5c243686d namespace: kserve-ci-e2e-test resourceVersion: "19133" uid: cafaec9a-3a70-4ae1-a7da-da25ea8522d4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn namespace: kserve-ci-e2e-test resourceVersion: "19090" uid: 4c6d57fc-3f16-4060-8fff-5279965ae841 kind: Event lastTimestamp: "2026-07-08T16:55:26Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.384s (1.384s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-08T16:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:26Z" name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn.18c05ed614c4ddc8 namespace: kserve-ci-e2e-test resourceVersion: "19148" uid: 5f26eb37-1d65-49e3-8a7b-13ae3bb93e03 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn namespace: kserve-ci-e2e-test resourceVersion: "19090" uid: 4c6d57fc-3f16-4060-8fff-5279965ae841 kind: Event lastTimestamp: "2026-07-08T16:55:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T16:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:26Z" name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn.18c05ed61824a2b0 namespace: kserve-ci-e2e-test resourceVersion: "19149" uid: ca3314c8-7f0c-432d-994b-f852fc8ae9fd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn namespace: kserve-ci-e2e-test resourceVersion: "19090" uid: 4c6d57fc-3f16-4060-8fff-5279965ae841 kind: Event lastTimestamp: "2026-07-08T16:55:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T16:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:26Z" name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn.18c05ed6189fca20 namespace: kserve-ci-e2e-test resourceVersion: "19150" uid: 98dbca8e-83ce-4624-9ca2-130da273dbee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn namespace: kserve-ci-e2e-test resourceVersion: "19090" uid: 4c6d57fc-3f16-4060-8fff-5279965ae841 kind: Event lastTimestamp: "2026-07-08T16:55:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T16:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:26Z" name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn.18c05ed618aa3cfa namespace: kserve-ci-e2e-test resourceVersion: "19151" uid: 4e5f1e57-eed2-4df6-a2eb-207c31b87fab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn namespace: kserve-ci-e2e-test resourceVersion: "19090" uid: 4c6d57fc-3f16-4060-8fff-5279965ae841 kind: Event lastTimestamp: "2026-07-08T16:55:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T16:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:26Z" name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn.18c05ed61c08254d namespace: kserve-ci-e2e-test resourceVersion: "19152" uid: a73be951-b13f-4f6e-b5ad-daade824242c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn namespace: kserve-ci-e2e-test resourceVersion: "19090" uid: 4c6d57fc-3f16-4060-8fff-5279965ae841 kind: Event lastTimestamp: "2026-07-08T16:55:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:26Z" name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn.18c05ed61c84748d namespace: kserve-ci-e2e-test resourceVersion: "19153" uid: 26e060ed-cd0a-425f-b073-ec1a876c961c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn namespace: kserve-ci-e2e-test resourceVersion: "19090" uid: 4c6d57fc-3f16-4060-8fff-5279965ae841 kind: Event lastTimestamp: "2026-07-08T16:57:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T16:57:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:09Z" name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn.18c05eee10897a1c namespace: kserve-ci-e2e-test resourceVersion: "20057" uid: b02b79ff-9ba4-4e95-9a15-670538279536 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:57:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn namespace: kserve-ci-e2e-test resourceVersion: "19090" uid: 4c6d57fc-3f16-4060-8fff-5279965ae841 kind: Event lastTimestamp: "2026-07-08T16:57:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T16:57:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:57:09Z" name: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn.18c05eee108a0883 namespace: kserve-ci-e2e-test resourceVersion: "20059" uid: 5c2973e8-16a6-4b51-98bd-1b00a39c934d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-9f040-predictor-5458f85b97 namespace: kserve-ci-e2e-test resourceVersion: "19075" uid: f66d219d-7bc4-49b8-8b8e-b91d5a8b1db1 kind: Event lastTimestamp: "2026-07-08T16:55:23Z" message: 'Created pod: message-dumper-raw-9f040-predictor-5458f85b97-zlgtn' metadata: creationTimestamp: "2026-07-08T16:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:23Z" name: message-dumper-raw-9f040-predictor-5458f85b97.18c05ed582f6dc5e namespace: kserve-ci-e2e-test resourceVersion: "19089" uid: a868ac4e-c420-4b39-984c-38b7d0ea607e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-9f040-predictor namespace: kserve-ci-e2e-test resourceVersion: "19067" uid: ecdd19ae-64da-4a80-887f-f8c83e3282b5 kind: Event lastTimestamp: "2026-07-08T16:55:23Z" message: Scaled up replica set message-dumper-raw-9f040-predictor-5458f85b97 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:23Z" name: message-dumper-raw-9f040-predictor.18c05ed57fc6287c namespace: kserve-ci-e2e-test resourceVersion: "19077" uid: 87897147-2210-4a82-a55a-d11b8f87a387 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:55:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-9f040-predictor namespace: kserve-ci-e2e-test resourceVersion: "19074" uid: 22243f88-ddf2-4101-a2a5-5f7b4acf1b9d kind: Event lastTimestamp: "2026-07-08T16:55:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:55:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:53Z" name: message-dumper-raw-9f040-predictor.18c05ed8fe6aa23b namespace: kserve-ci-e2e-test resourceVersion: "19440" uid: 1bf96c26-1537-4b93-89c1-4546dd83e9a9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T16:55:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-9f040-predictor namespace: kserve-ci-e2e-test resourceVersion: "19074" uid: 22243f88-ddf2-4101-a2a5-5f7b4acf1b9d kind: Event lastTimestamp: "2026-07-08T16:55:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T16:55:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:53Z" name: message-dumper-raw-9f040-predictor.18c05ed8fec1c67d namespace: kserve-ci-e2e-test resourceVersion: "19441" uid: 0ef1f2ae-49e4-4bf5-b752-4a2a53f38425 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-08T16:55:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-9f040 namespace: kserve-ci-e2e-test resourceVersion: "19052" uid: c2902b5d-9f37-47d7-aaa4-8d0adb75e776 kind: Event lastTimestamp: "2026-07-08T16:55:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T16:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:55:33Z" name: message-dumper-raw-9f040.18c05ed57ff8c33f namespace: kserve-ci-e2e-test resourceVersion: "19216" uid: 4a9e8471-3ecd-4395-afee-e5b6dbe12ae3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-9f040 namespace: kserve-ci-e2e-test resourceVersion: "19052" uid: c2902b5d-9f37-47d7-aaa4-8d0adb75e776 kind: Event lastTimestamp: "2026-07-08T16:55:23Z" message: 'Failed to update status for InferenceService "message-dumper-raw-9f040": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-9f040": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T16:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:55:23Z" name: message-dumper-raw-9f040.18c05ed5806135fd namespace: kserve-ci-e2e-test resourceVersion: "19082" uid: 297035a2-a375-4767-8ade-09b6e8ede475 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-9f040 namespace: kserve-ci-e2e-test resourceVersion: "19213" uid: c2902b5d-9f37-47d7-aaa4-8d0adb75e776 kind: Event lastTimestamp: "2026-07-08T16:55:33Z" message: InferenceService [message-dumper-raw-9f040] is Ready metadata: creationTimestamp: "2026-07-08T16:55:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:55:33Z" name: message-dumper-raw-9f040.18c05ed7c6ec3ac7 namespace: kserve-ci-e2e-test resourceVersion: "19215" uid: 4cff3485-8747-4381-8390-78d1def89ef8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:53:22.968383Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-5d281-676947cfc6-6zlv7 namespace: kserve-ci-e2e-test resourceVersion: "17877" uid: cf8227f9-ee3c-4297-94e4-b44f4f89c48b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-5d281-676947cfc6-6zlv7 to ip-10-0-131-106.ec2.internal metadata: creationTimestamp: "2026-07-08T16:53:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:53:22Z" name: model-chainer-raw-5d281-676947cfc6-6zlv7.18c05eb96f5317a0 namespace: kserve-ci-e2e-test resourceVersion: "17882" uid: 3ecc7ae1-1ee3-4dbe-8e67-476815c6aa9b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd6c76d55-gncps source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:23Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-5d281-676947cfc6-6zlv7 namespace: kserve-ci-e2e-test resourceVersion: "17885" uid: cf8227f9-ee3c-4297-94e4-b44f4f89c48b kind: Event lastTimestamp: "2026-07-08T16:53:23Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:53:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:53:23Z" name: model-chainer-raw-5d281-676947cfc6-6zlv7.18c05eb98913490e namespace: kserve-ci-e2e-test resourceVersion: "17896" uid: 03680382-3c0e-448c-9295-cbd27c9df0fd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-5d281} kind: Pod name: model-chainer-raw-5d281-676947cfc6-6zlv7 namespace: kserve-ci-e2e-test resourceVersion: "17881" uid: cf8227f9-ee3c-4297-94e4-b44f4f89c48b kind: Event lastTimestamp: "2026-07-08T16:53:23Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:95154c3f09f98c64e7d79b759f3f17a91001c79b4e6c56ac6d0a6d8b1e47cf38" metadata: creationTimestamp: "2026-07-08T16:53:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:23Z" name: model-chainer-raw-5d281-676947cfc6-6zlv7.18c05eb98b6e5b03 namespace: kserve-ci-e2e-test resourceVersion: "17898" uid: 9fc60fe9-128e-4f25-9a3f-3aef70cdbac1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-106.ec2.internal source: component: kubelet host: ip-10-0-131-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-5d281} kind: Pod name: model-chainer-raw-5d281-676947cfc6-6zlv7 namespace: kserve-ci-e2e-test resourceVersion: "17881" uid: cf8227f9-ee3c-4297-94e4-b44f4f89c48b kind: Event lastTimestamp: "2026-07-08T16:53:25Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:95154c3f09f98c64e7d79b759f3f17a91001c79b4e6c56ac6d0a6d8b1e47cf38" in 2.312s (2.312s including waiting). Image size: 218072672 bytes.' metadata: creationTimestamp: "2026-07-08T16:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:25Z" name: model-chainer-raw-5d281-676947cfc6-6zlv7.18c05eba153d72e8 namespace: kserve-ci-e2e-test resourceVersion: "17913" uid: 8a461521-0151-4868-b183-f45c96f28f0e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-106.ec2.internal source: component: kubelet host: ip-10-0-131-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-5d281} kind: Pod name: model-chainer-raw-5d281-676947cfc6-6zlv7 namespace: kserve-ci-e2e-test resourceVersion: "17881" uid: cf8227f9-ee3c-4297-94e4-b44f4f89c48b kind: Event lastTimestamp: "2026-07-08T16:53:25Z" message: 'Created container: model-chainer-raw-5d281' metadata: creationTimestamp: "2026-07-08T16:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:25Z" name: model-chainer-raw-5d281-676947cfc6-6zlv7.18c05eba18f400aa namespace: kserve-ci-e2e-test resourceVersion: "17914" uid: 769e9969-4bdf-4dce-9829-147ec7a764da reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-106.ec2.internal source: component: kubelet host: ip-10-0-131-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-5d281} kind: Pod name: model-chainer-raw-5d281-676947cfc6-6zlv7 namespace: kserve-ci-e2e-test resourceVersion: "17881" uid: cf8227f9-ee3c-4297-94e4-b44f4f89c48b kind: Event lastTimestamp: "2026-07-08T16:53:25Z" message: Started container model-chainer-raw-5d281 metadata: creationTimestamp: "2026-07-08T16:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:25Z" name: model-chainer-raw-5d281-676947cfc6-6zlv7.18c05eba19677af3 namespace: kserve-ci-e2e-test resourceVersion: "17915" uid: faf3a78e-9135-4d0d-b119-61131fabb8cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-106.ec2.internal source: component: kubelet host: ip-10-0-131-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-5d281} kind: Pod name: model-chainer-raw-5d281-676947cfc6-6zlv7 namespace: kserve-ci-e2e-test resourceVersion: "17881" uid: cf8227f9-ee3c-4297-94e4-b44f4f89c48b kind: Event lastTimestamp: "2026-07-08T16:53:32Z" message: Stopping container model-chainer-raw-5d281 metadata: creationTimestamp: "2026-07-08T16:53:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:53:32Z" name: model-chainer-raw-5d281-676947cfc6-6zlv7.18c05ebbc46e6c99 namespace: kserve-ci-e2e-test resourceVersion: "17991" uid: 438729fc-48b4-49b7-9c98-aafd53c0b9f4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-106.ec2.internal source: component: kubelet host: ip-10-0-131-106.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:53:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-5d281} kind: Pod name: model-chainer-raw-5d281-676947cfc6-6zlv7 namespace: kserve-ci-e2e-test resourceVersion: "17881" uid: cf8227f9-ee3c-4297-94e4-b44f4f89c48b kind: Event lastTimestamp: "2026-07-08T16:54:02Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T16:53:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:54:02Z" name: model-chainer-raw-5d281-676947cfc6-6zlv7.18c05ebcbba84cfd namespace: kserve-ci-e2e-test resourceVersion: "18377" uid: 9cb6526e-2ad2-4a89-93c3-d6e9b586a5cc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-106.ec2.internal source: component: kubelet host: ip-10-0-131-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-5d281-676947cfc6 namespace: kserve-ci-e2e-test resourceVersion: "17873" uid: c6e31afa-6563-44a2-9bcd-b72fa2c6acc2 kind: Event lastTimestamp: "2026-07-08T16:53:22Z" message: 'Created pod: model-chainer-raw-5d281-676947cfc6-6zlv7' metadata: creationTimestamp: "2026-07-08T16:53:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:53:22Z" name: model-chainer-raw-5d281-676947cfc6.18c05eb96e695981 namespace: kserve-ci-e2e-test resourceVersion: "17879" uid: d7dda7a4-6e9e-427b-9b73-e680502b9545 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-5d281 namespace: kserve-ci-e2e-test resourceVersion: "17870" uid: de781a17-9bf7-4517-8bfd-9dd7e4910e29 kind: Event lastTimestamp: "2026-07-08T16:53:22Z" message: Scaled up replica set model-chainer-raw-5d281-676947cfc6 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:53:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:53:22Z" name: model-chainer-raw-5d281.18c05eb96cd44a60 namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: 96d75883-16f1-4db9-8e6f-58ee0eda8783 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:53:23Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-5d281 namespace: kserve-ci-e2e-test resourceVersion: "17889" uid: ad4d543a-d952-46d8-8151-bd8640d8c575 kind: Event lastTimestamp: "2026-07-08T16:53:23Z" message: InferenceGraph [model-chainer-raw-5d281] is Ready metadata: creationTimestamp: "2026-07-08T16:53:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:53:23Z" name: model-chainer-raw-5d281.18c05eb976a7def9 namespace: kserve-ci-e2e-test resourceVersion: "17891" uid: db15f0bd-31e3-495a-b0cf-258726ba9891 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T16:55:13.297190Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-3e871-578944cc48-rhd7n namespace: kserve-ci-e2e-test resourceVersion: "18924" uid: 3dfe0c93-941a-477c-adba-484df85df76a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-3e871-578944cc48-rhd7n to ip-10-0-131-106.ec2.internal metadata: creationTimestamp: "2026-07-08T16:55:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T16:55:13Z" name: model-chainer-raw-hpa-3e871-578944cc48-rhd7n.18c05ed31f6f0513 namespace: kserve-ci-e2e-test resourceVersion: "18928" uid: 49a1971e-e773-4651-8393-1f634901166d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd6c76d55-gncps source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:13Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-3e871-578944cc48-rhd7n namespace: kserve-ci-e2e-test resourceVersion: "18927" uid: 3dfe0c93-941a-477c-adba-484df85df76a kind: Event lastTimestamp: "2026-07-08T16:55:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-3e871-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T16:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:13Z" name: model-chainer-raw-hpa-3e871-578944cc48-rhd7n.18c05ed32dc8432f namespace: kserve-ci-e2e-test resourceVersion: "18941" uid: e3750aa0-3912-4135-bd4c-e5d68d58d7b9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-106.ec2.internal source: component: kubelet host: ip-10-0-131-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:14Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-3e871-578944cc48-rhd7n namespace: kserve-ci-e2e-test resourceVersion: "18932" uid: 3dfe0c93-941a-477c-adba-484df85df76a kind: Event lastTimestamp: "2026-07-08T16:55:14Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T16:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T16:55:14Z" name: model-chainer-raw-hpa-3e871-578944cc48-rhd7n.18c05ed35cbd3893 namespace: kserve-ci-e2e-test resourceVersion: "18948" uid: 4f9cf61a-7b10-4a85-b9d2-71b00dfd5b95 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3e871} kind: Pod name: model-chainer-raw-hpa-3e871-578944cc48-rhd7n namespace: kserve-ci-e2e-test resourceVersion: "18927" uid: 3dfe0c93-941a-477c-adba-484df85df76a kind: Event lastTimestamp: "2026-07-08T16:55:14Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:95154c3f09f98c64e7d79b759f3f17a91001c79b4e6c56ac6d0a6d8b1e47cf38" already present on machine metadata: creationTimestamp: "2026-07-08T16:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:14Z" name: model-chainer-raw-hpa-3e871-578944cc48-rhd7n.18c05ed35df23832 namespace: kserve-ci-e2e-test resourceVersion: "18950" uid: 8341bb98-2422-43c7-9c11-d356393f6d40 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-106.ec2.internal source: component: kubelet host: ip-10-0-131-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3e871} kind: Pod name: model-chainer-raw-hpa-3e871-578944cc48-rhd7n namespace: kserve-ci-e2e-test resourceVersion: "18927" uid: 3dfe0c93-941a-477c-adba-484df85df76a kind: Event lastTimestamp: "2026-07-08T16:55:14Z" message: 'Created container: model-chainer-raw-hpa-3e871' metadata: creationTimestamp: "2026-07-08T16:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:14Z" name: model-chainer-raw-hpa-3e871-578944cc48-rhd7n.18c05ed360e53a9a namespace: kserve-ci-e2e-test resourceVersion: "18951" uid: ff9ad928-641e-4f72-81be-b0912a572590 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-106.ec2.internal source: component: kubelet host: ip-10-0-131-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3e871} kind: Pod name: model-chainer-raw-hpa-3e871-578944cc48-rhd7n namespace: kserve-ci-e2e-test resourceVersion: "18927" uid: 3dfe0c93-941a-477c-adba-484df85df76a kind: Event lastTimestamp: "2026-07-08T16:55:14Z" message: Started container model-chainer-raw-hpa-3e871 metadata: creationTimestamp: "2026-07-08T16:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:14Z" name: model-chainer-raw-hpa-3e871-578944cc48-rhd7n.18c05ed361502db1 namespace: kserve-ci-e2e-test resourceVersion: "18952" uid: d586c5fa-2a51-468f-8db3-af607471e8eb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-106.ec2.internal source: component: kubelet host: ip-10-0-131-106.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3e871} kind: Pod name: model-chainer-raw-hpa-3e871-578944cc48-rhd7n namespace: kserve-ci-e2e-test resourceVersion: "18927" uid: 3dfe0c93-941a-477c-adba-484df85df76a kind: Event lastTimestamp: "2026-07-08T16:55:23Z" message: Stopping container model-chainer-raw-hpa-3e871 metadata: creationTimestamp: "2026-07-08T16:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:23Z" name: model-chainer-raw-hpa-3e871-578944cc48-rhd7n.18c05ed576dae5a5 namespace: kserve-ci-e2e-test resourceVersion: "19044" uid: 850b7ccf-5b1c-4492-ab84-4030b05489f3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-106.ec2.internal source: component: kubelet host: ip-10-0-131-106.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-08T16:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-3e871} kind: Pod name: model-chainer-raw-hpa-3e871-578944cc48-rhd7n namespace: kserve-ci-e2e-test resourceVersion: "18927" uid: 3dfe0c93-941a-477c-adba-484df85df76a kind: Event lastTimestamp: "2026-07-08T16:55:50Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-08T16:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T16:55:50Z" name: model-chainer-raw-hpa-3e871-578944cc48-rhd7n.18c05ed5ff44304c namespace: kserve-ci-e2e-test resourceVersion: "19417" uid: e49d12ca-4fc8-4bd9-9e7d-931a50a8e96c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-106.ec2.internal source: component: kubelet host: ip-10-0-131-106.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-3e871-578944cc48 namespace: kserve-ci-e2e-test resourceVersion: "18920" uid: e0be07be-8f0c-4a16-b712-f96aba1d2d23 kind: Event lastTimestamp: "2026-07-08T16:55:13Z" message: 'Created pod: model-chainer-raw-hpa-3e871-578944cc48-rhd7n' metadata: creationTimestamp: "2026-07-08T16:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:13Z" name: model-chainer-raw-hpa-3e871-578944cc48.18c05ed31ec63d89 namespace: kserve-ci-e2e-test resourceVersion: "18925" uid: f273b55a-98fd-44bd-95c7-d9c3fee5cd33 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-3e871 namespace: kserve-ci-e2e-test resourceVersion: "18917" uid: e4214ba8-33d1-4b91-934a-ede6b93b410f kind: Event lastTimestamp: "2026-07-08T16:55:13Z" message: Scaled up replica set model-chainer-raw-hpa-3e871-578944cc48 from 0 to 1 metadata: creationTimestamp: "2026-07-08T16:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T16:55:13Z" name: model-chainer-raw-hpa-3e871.18c05ed31d7e11ea namespace: kserve-ci-e2e-test resourceVersion: "18921" uid: d24b0ced-dc05-42eb-a53f-074a082f8c96 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T16:55:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-3e871 namespace: kserve-ci-e2e-test resourceVersion: "18936" uid: bfffcd35-52f8-4220-a248-9472e7fff5e9 kind: Event lastTimestamp: "2026-07-08T16:55:13Z" message: InferenceGraph [model-chainer-raw-hpa-3e871] is Ready metadata: creationTimestamp: "2026-07-08T16:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T16:55:13Z" name: model-chainer-raw-hpa-3e871.18c05ed324253465 namespace: kserve-ci-e2e-test resourceVersion: "18938" uid: 8cdeae40-e593-411b-b66c-fdd93a736719 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:02:01.041803Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-f05f8-predictor-54888985c8-gwssx namespace: kserve-ci-e2e-test resourceVersion: "22470" uid: e5afccf1-df32-44fb-9ce5-e0487e67eaef kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-f05f8-predictor-54888985c8-gwssx to ip-10-0-142-94.ec2.internal metadata: creationTimestamp: "2026-07-08T17:02:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:02:01Z" name: raw-sklearn-f05f8-predictor-54888985c8-gwssx.18c05f320ee820d6 namespace: kserve-ci-e2e-test resourceVersion: "22476" uid: a8e2fa7a-3858-4c42-bcb6-89a7658e3e20 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd6c76d55-gncps source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:02:01Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-f05f8-predictor-54888985c8-gwssx namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: e5afccf1-df32-44fb-9ce5-e0487e67eaef kind: Event lastTimestamp: "2026-07-08T17:02:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-f05f8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-08T17:02:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:02:01Z" name: raw-sklearn-f05f8-predictor-54888985c8-gwssx.18c05f321d84bf71 namespace: kserve-ci-e2e-test resourceVersion: "22486" uid: 56a83621-edf0-4cad-ab26-1c261e8f704f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:02:02Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-f05f8-predictor-54888985c8-gwssx namespace: kserve-ci-e2e-test resourceVersion: "22482" uid: e5afccf1-df32-44fb-9ce5-e0487e67eaef kind: Event lastTimestamp: "2026-07-08T17:02:02Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:02:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:02:02Z" name: raw-sklearn-f05f8-predictor-54888985c8-gwssx.18c05f324ccebef6 namespace: kserve-ci-e2e-test resourceVersion: "22494" uid: 09963c88-1e93-4031-9889-34f6b20a5854 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:02:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-f05f8-predictor-54888985c8-gwssx namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: e5afccf1-df32-44fb-9ce5-e0487e67eaef kind: Event lastTimestamp: "2026-07-08T17:02:02Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7a9c075ae5e64d970cc026f71d8348f193fda0ed6de6807045e29c208821be1e" already present on machine metadata: creationTimestamp: "2026-07-08T17:02:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:02:02Z" name: raw-sklearn-f05f8-predictor-54888985c8-gwssx.18c05f324e4b2f19 namespace: kserve-ci-e2e-test resourceVersion: "22497" uid: a260e616-94f9-4c57-8287-1bb6d888ea2c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:02:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-f05f8-predictor-54888985c8-gwssx namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: e5afccf1-df32-44fb-9ce5-e0487e67eaef kind: Event lastTimestamp: "2026-07-08T17:02:02Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T17:02:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:02:02Z" name: raw-sklearn-f05f8-predictor-54888985c8-gwssx.18c05f3250dec83c namespace: kserve-ci-e2e-test resourceVersion: "22498" uid: c1775b45-fce6-4ecc-91d4-ec9611dc40e6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:02:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-f05f8-predictor-54888985c8-gwssx namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: e5afccf1-df32-44fb-9ce5-e0487e67eaef kind: Event lastTimestamp: "2026-07-08T17:02:02Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T17:02:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:02:02Z" name: raw-sklearn-f05f8-predictor-54888985c8-gwssx.18c05f32515a2ce4 namespace: kserve-ci-e2e-test resourceVersion: "22499" uid: 2e5d5c0d-2a0b-4868-8b72-ca01463c3515 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:02:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-f05f8-predictor-54888985c8-gwssx namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: e5afccf1-df32-44fb-9ce5-e0487e67eaef kind: Event lastTimestamp: "2026-07-08T17:02:06Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T17:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:02:06Z" name: raw-sklearn-f05f8-predictor-54888985c8-gwssx.18c05f3368db5404 namespace: kserve-ci-e2e-test resourceVersion: "22536" uid: 665784c0-d4fa-480d-acb4-cfc5b3e3aa48 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:02:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-f05f8-predictor-54888985c8-gwssx namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: e5afccf1-df32-44fb-9ce5-e0487e67eaef kind: Event lastTimestamp: "2026-07-08T17:02:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T17:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:02:06Z" name: raw-sklearn-f05f8-predictor-54888985c8-gwssx.18c05f336c009c42 namespace: kserve-ci-e2e-test resourceVersion: "22538" uid: 615e4b1f-dd34-4230-89ea-75dc2c23c464 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:02:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-f05f8-predictor-54888985c8-gwssx namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: e5afccf1-df32-44fb-9ce5-e0487e67eaef kind: Event lastTimestamp: "2026-07-08T17:02:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T17:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:02:06Z" name: raw-sklearn-f05f8-predictor-54888985c8-gwssx.18c05f336c7daf87 namespace: kserve-ci-e2e-test resourceVersion: "22540" uid: 9293fd54-7f88-460e-9826-8f2148157457 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:02:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-f05f8-predictor-54888985c8-gwssx namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: e5afccf1-df32-44fb-9ce5-e0487e67eaef kind: Event lastTimestamp: "2026-07-08T17:02:06Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T17:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:02:06Z" name: raw-sklearn-f05f8-predictor-54888985c8-gwssx.18c05f336c8a2637 namespace: kserve-ci-e2e-test resourceVersion: "22542" uid: 5d132672-e6fc-4aee-8453-6c83c8e237a5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:02:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-f05f8-predictor-54888985c8-gwssx namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: e5afccf1-df32-44fb-9ce5-e0487e67eaef kind: Event lastTimestamp: "2026-07-08T17:02:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T17:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:02:06Z" name: raw-sklearn-f05f8-predictor-54888985c8-gwssx.18c05f336f98527f namespace: kserve-ci-e2e-test resourceVersion: "22543" uid: 6ad5749d-1ed6-484a-8851-fe9b6ec62359 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:02:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-f05f8-predictor-54888985c8-gwssx namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: e5afccf1-df32-44fb-9ce5-e0487e67eaef kind: Event lastTimestamp: "2026-07-08T17:02:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:02:06Z" name: raw-sklearn-f05f8-predictor-54888985c8-gwssx.18c05f3370150621 namespace: kserve-ci-e2e-test resourceVersion: "22545" uid: fbf25479-ad25-4ea1-895c-ba75064c70b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T17:02:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-f05f8-predictor-54888985c8-gwssx namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: e5afccf1-df32-44fb-9ce5-e0487e67eaef kind: Event lastTimestamp: "2026-07-08T17:03:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:03:33Z" name: raw-sklearn-f05f8-predictor-54888985c8-gwssx.18c05f33a4d9a5df namespace: kserve-ci-e2e-test resourceVersion: "23190" uid: 1a9d6d67-a914-40e4-8647-65e956888240 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:03:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-f05f8-predictor-54888985c8-gwssx namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: e5afccf1-df32-44fb-9ce5-e0487e67eaef kind: Event lastTimestamp: "2026-07-08T17:03:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:03:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:03:31Z" name: raw-sklearn-f05f8-predictor-54888985c8-gwssx.18c05f470b086927 namespace: kserve-ci-e2e-test resourceVersion: "23164" uid: 4492c7b5-74a4-4909-9281-42374ea76256 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:03:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-f05f8-predictor-54888985c8-gwssx namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: e5afccf1-df32-44fb-9ce5-e0487e67eaef kind: Event lastTimestamp: "2026-07-08T17:03:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:03:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:03:31Z" name: raw-sklearn-f05f8-predictor-54888985c8-gwssx.18c05f470b090396 namespace: kserve-ci-e2e-test resourceVersion: "23165" uid: 06afe22c-7d1a-48d9-92f4-e8a9a6649872 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:03:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-f05f8-predictor-54888985c8-gwssx namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: e5afccf1-df32-44fb-9ce5-e0487e67eaef kind: Event lastTimestamp: "2026-07-08T17:03:33Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:03:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:03:33Z" name: raw-sklearn-f05f8-predictor-54888985c8-gwssx.18c05f47ab14bdeb namespace: kserve-ci-e2e-test resourceVersion: "23189" uid: f76a5a50-7ded-4fd3-b82f-8c16898879c7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:02:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-f05f8-predictor-54888985c8 namespace: kserve-ci-e2e-test resourceVersion: "22462" uid: a1fdb997-2665-4ee9-8cc7-4ac8c066ebdd kind: Event lastTimestamp: "2026-07-08T17:02:01Z" message: 'Created pod: raw-sklearn-f05f8-predictor-54888985c8-gwssx' metadata: creationTimestamp: "2026-07-08T17:02:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:02:01Z" name: raw-sklearn-f05f8-predictor-54888985c8.18c05f320e60f0ee namespace: kserve-ci-e2e-test resourceVersion: "22473" uid: 6fdb6dea-2b43-4f4d-9508-0dcbbc61c921 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:02:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-f05f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22460" uid: 116f598d-4baf-4b3c-945c-78ad08d0ee50 kind: Event lastTimestamp: "2026-07-08T17:02:00Z" message: Scaled up replica set raw-sklearn-f05f8-predictor-54888985c8 from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:02:00Z" name: raw-sklearn-f05f8-predictor.18c05f320b15be24 namespace: kserve-ci-e2e-test resourceVersion: "22463" uid: a073d546-3dbf-49ee-8fc5-8b5e663ab541 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T17:02:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-f05f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22461" uid: 2b25f81c-8ba2-4376-a6d2-75a1ae4f9a27 kind: Event lastTimestamp: "2026-07-08T17:02:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:02:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:02:46Z" name: raw-sklearn-f05f8-predictor.18c05f3589885704 namespace: kserve-ci-e2e-test resourceVersion: "22824" uid: 50496e3f-cf6b-4888-ba67-2a297acfa394 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T17:02:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-f05f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22461" uid: 2b25f81c-8ba2-4376-a6d2-75a1ae4f9a27 kind: Event lastTimestamp: "2026-07-08T17:02:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:02:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:02:46Z" name: raw-sklearn-f05f8-predictor.18c05f3589f0b1c1 namespace: kserve-ci-e2e-test resourceVersion: "22825" uid: 0b33e419-c5e6-4137-93e4-0e4d170df422 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T17:03:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-f05f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22615" uid: 2b25f81c-8ba2-4376-a6d2-75a1ae4f9a27 kind: Event lastTimestamp: "2026-07-08T17:03:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:03:31Z" name: raw-sklearn-f05f8-predictor.18c05f4007229e13 namespace: kserve-ci-e2e-test resourceVersion: "23147" uid: 835f43d2-cd37-416b-866e-434adea2f5dd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-08T17:03:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-f05f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22615" uid: 2b25f81c-8ba2-4376-a6d2-75a1ae4f9a27 kind: Event lastTimestamp: "2026-07-08T17:03:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:03:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:03:31Z" name: raw-sklearn-f05f8-predictor.18c05f4007943a21 namespace: kserve-ci-e2e-test resourceVersion: "23148" uid: 8ca1aff8-ac92-4502-b275-f2c2195fad47 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-08T17:02:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-f05f8 namespace: kserve-ci-e2e-test resourceVersion: "22453" uid: c3cd8994-f061-4e2e-8d8c-d9fa6a47990c kind: Event lastTimestamp: "2026-07-08T17:03:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T17:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:03:23Z" name: raw-sklearn-f05f8.18c05f320b1b4e3c namespace: kserve-ci-e2e-test resourceVersion: "23097" uid: bf7fc237-335c-42c9-837b-b219be95bf52 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:02:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-f05f8 namespace: kserve-ci-e2e-test resourceVersion: "22453" uid: c3cd8994-f061-4e2e-8d8c-d9fa6a47990c kind: Event lastTimestamp: "2026-07-08T17:02:00Z" message: 'Failed to update status for InferenceService "raw-sklearn-f05f8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-f05f8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:02:00Z" name: raw-sklearn-f05f8.18c05f320b72fedf namespace: kserve-ci-e2e-test resourceVersion: "22466" uid: 287c627e-5ef5-47cd-8119-982adcdc9a9e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:03:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-f05f8 namespace: kserve-ci-e2e-test resourceVersion: "23095" uid: c3cd8994-f061-4e2e-8d8c-d9fa6a47990c kind: Event lastTimestamp: "2026-07-08T17:03:23Z" message: InferenceService [raw-sklearn-f05f8] is Ready metadata: creationTimestamp: "2026-07-08T17:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:03:23Z" name: raw-sklearn-f05f8.18c05f455c974827 namespace: kserve-ci-e2e-test resourceVersion: "23096" uid: e877c52b-32ab-4b47-891f-5e6ca79e5046 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-08T17:03:36.371201Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9 namespace: kserve-ci-e2e-test resourceVersion: "23234" uid: b906914b-10ad-4131-bd22-ef6bf3e03d68 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9 to ip-10-0-142-94.ec2.internal metadata: creationTimestamp: "2026-07-08T17:03:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-08T17:03:36Z" name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9.18c05f4840fb38af namespace: kserve-ci-e2e-test resourceVersion: "23238" uid: 839af567-6f41-4d14-8cd7-d69aa9594b01 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5fd6c76d55-gncps source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:03:36Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9 namespace: kserve-ci-e2e-test resourceVersion: "23242" uid: b906914b-10ad-4131-bd22-ef6bf3e03d68 kind: Event lastTimestamp: "2026-07-08T17:03:36Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-08T17:03:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-08T17:03:36Z" name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9.18c05f485abbfb81 namespace: kserve-ci-e2e-test resourceVersion: "23249" uid: a886d42e-2ecd-44a4-be9f-b58109e01654 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:03:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9 namespace: kserve-ci-e2e-test resourceVersion: "23235" uid: b906914b-10ad-4131-bd22-ef6bf3e03d68 kind: Event lastTimestamp: "2026-07-08T17:03:36Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7a9c075ae5e64d970cc026f71d8348f193fda0ed6de6807045e29c208821be1e" already present on machine metadata: creationTimestamp: "2026-07-08T17:03:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:03:36Z" name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9.18c05f485c42475a namespace: kserve-ci-e2e-test resourceVersion: "23252" uid: c7df7b8b-cfe0-4903-8e62-12e2ec636a61 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:03:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9 namespace: kserve-ci-e2e-test resourceVersion: "23235" uid: b906914b-10ad-4131-bd22-ef6bf3e03d68 kind: Event lastTimestamp: "2026-07-08T17:03:36Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-08T17:03:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:03:36Z" name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9.18c05f485fb68ecc namespace: kserve-ci-e2e-test resourceVersion: "23253" uid: 92915fb9-0941-4e76-8145-b14086240e87 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:03:36Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9 namespace: kserve-ci-e2e-test resourceVersion: "23235" uid: b906914b-10ad-4131-bd22-ef6bf3e03d68 kind: Event lastTimestamp: "2026-07-08T17:03:36Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-08T17:03:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:03:36Z" name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9.18c05f48602a5dc6 namespace: kserve-ci-e2e-test resourceVersion: "23254" uid: 5ff7c415-bc0a-4bd4-9cf5-63822a0cc6b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:03:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9 namespace: kserve-ci-e2e-test resourceVersion: "23235" uid: b906914b-10ad-4131-bd22-ef6bf3e03d68 kind: Event lastTimestamp: "2026-07-08T17:03:41Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1711" already present on machine metadata: creationTimestamp: "2026-07-08T17:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:03:41Z" name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9.18c05f49626af34a namespace: kserve-ci-e2e-test resourceVersion: "23295" uid: 681582f0-9a0b-42f1-95dd-d541fa77d237 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:03:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9 namespace: kserve-ci-e2e-test resourceVersion: "23235" uid: b906914b-10ad-4131-bd22-ef6bf3e03d68 kind: Event lastTimestamp: "2026-07-08T17:03:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-08T17:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:03:41Z" name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9.18c05f49668a50ca namespace: kserve-ci-e2e-test resourceVersion: "23299" uid: 4d7b313f-6aeb-432e-9328-ca07790840df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:03:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9 namespace: kserve-ci-e2e-test resourceVersion: "23235" uid: b906914b-10ad-4131-bd22-ef6bf3e03d68 kind: Event lastTimestamp: "2026-07-08T17:03:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-08T17:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:03:41Z" name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9.18c05f4967017f2d namespace: kserve-ci-e2e-test resourceVersion: "23300" uid: 0c3ae11a-29a9-42b8-bf7b-b9d5feb0d40f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:03:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9 namespace: kserve-ci-e2e-test resourceVersion: "23235" uid: b906914b-10ad-4131-bd22-ef6bf3e03d68 kind: Event lastTimestamp: "2026-07-08T17:03:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-08T17:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:03:41Z" name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9.18c05f49670b1e30 namespace: kserve-ci-e2e-test resourceVersion: "23301" uid: 84b362d4-24a9-415e-8b20-fcefaf1f10fb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:03:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9 namespace: kserve-ci-e2e-test resourceVersion: "23235" uid: b906914b-10ad-4131-bd22-ef6bf3e03d68 kind: Event lastTimestamp: "2026-07-08T17:03:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-08T17:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:03:41Z" name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9.18c05f496cc3c2fb namespace: kserve-ci-e2e-test resourceVersion: "23303" uid: 9229e954-83d2-4ba9-9c51-0a3a099e40dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:03:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9 namespace: kserve-ci-e2e-test resourceVersion: "23235" uid: b906914b-10ad-4131-bd22-ef6bf3e03d68 kind: Event lastTimestamp: "2026-07-08T17:03:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:03:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:03:41Z" name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9.18c05f496d3fc91b namespace: kserve-ci-e2e-test resourceVersion: "23304" uid: 79c330e7-da87-4c37-b9a8-ec41cc17f850 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T17:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9 namespace: kserve-ci-e2e-test resourceVersion: "23235" uid: b906914b-10ad-4131-bd22-ef6bf3e03d68 kind: Event lastTimestamp: "2026-07-08T17:05:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:08Z" name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9.18c05f499e6f9c50 namespace: kserve-ci-e2e-test resourceVersion: "24229" uid: 7d9aa00a-4028-44a6-8d61-a669378c3038 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9 namespace: kserve-ci-e2e-test resourceVersion: "23235" uid: b906914b-10ad-4131-bd22-ef6bf3e03d68 kind: Event lastTimestamp: "2026-07-08T17:05:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-08T17:05:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:06Z" name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9.18c05f5d3bca8ae2 namespace: kserve-ci-e2e-test resourceVersion: "24213" uid: a03aeaac-ff8b-4acb-a7d3-fec3e3316a57 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9 namespace: kserve-ci-e2e-test resourceVersion: "23235" uid: b906914b-10ad-4131-bd22-ef6bf3e03d68 kind: Event lastTimestamp: "2026-07-08T17:05:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-08T17:05:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:06Z" name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9.18c05f5d3bcbb3e5 namespace: kserve-ci-e2e-test resourceVersion: "24214" uid: af904fd3-e1c4-4ef5-88cf-a377c874f5c6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:05:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9 namespace: kserve-ci-e2e-test resourceVersion: "23235" uid: b906914b-10ad-4131-bd22-ef6bf3e03d68 kind: Event lastTimestamp: "2026-07-08T17:05:08Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-08T17:05:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-08T17:05:08Z" name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9.18c05f5da49d1e87 namespace: kserve-ci-e2e-test resourceVersion: "24228" uid: eb75f1e9-a97d-42bd-9fc1-20ecd87684a9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-94.ec2.internal source: component: kubelet host: ip-10-0-142-94.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:03:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8 namespace: kserve-ci-e2e-test resourceVersion: "23221" uid: 55028c67-7a6a-4748-b50b-3b95ab9dae53 kind: Event lastTimestamp: "2026-07-08T17:03:36Z" message: 'Created pod: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8-kjtk9' metadata: creationTimestamp: "2026-07-08T17:03:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:03:36Z" name: raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8.18c05f484097967e namespace: kserve-ci-e2e-test resourceVersion: "23236" uid: 6d28867e-6463-4762-b133-4203694a28c1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:03:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-cfbd9-predictor namespace: kserve-ci-e2e-test resourceVersion: "23219" uid: 434d4d35-809b-41fb-b2c4-1013101a8922 kind: Event lastTimestamp: "2026-07-08T17:03:36Z" message: Scaled up replica set raw-sklearn-runtime-cfbd9-predictor-7ccbb7c7f8 from 0 to 1 metadata: creationTimestamp: "2026-07-08T17:03:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:03:36Z" name: raw-sklearn-runtime-cfbd9-predictor.18c05f483bc132e0 namespace: kserve-ci-e2e-test resourceVersion: "23222" uid: 0a99c52b-9dd3-460a-8047-18feb547423e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:03:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-cfbd9-predictor namespace: kserve-ci-e2e-test resourceVersion: "23220" uid: 74cb0607-7b29-4636-a6bf-1659f05900b5 kind: Event lastTimestamp: "2026-07-08T17:04:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:04:06Z" name: raw-sklearn-runtime-cfbd9-predictor.18c05f4bba98e884 namespace: kserve-ci-e2e-test resourceVersion: "23482" uid: 3ac77983-f3eb-4a6b-9015-75b422e45b30 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-08T17:03:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-cfbd9-predictor namespace: kserve-ci-e2e-test resourceVersion: "23220" uid: 74cb0607-7b29-4636-a6bf-1659f05900b5 kind: Event lastTimestamp: "2026-07-08T17:04:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-08T17:03:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:04:06Z" name: raw-sklearn-runtime-cfbd9-predictor.18c05f4bbaf6cf0e namespace: kserve-ci-e2e-test resourceVersion: "23483" uid: dd0d2a14-638b-44d7-b8e2-dfcb70e7a912 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T17:04:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-cfbd9-predictor namespace: kserve-ci-e2e-test resourceVersion: "23373" uid: 74cb0607-7b29-4636-a6bf-1659f05900b5 kind: Event lastTimestamp: "2026-07-08T17:05:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:04:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:05:06Z" name: raw-sklearn-runtime-cfbd9-predictor.18c05f52b8d1d401 namespace: kserve-ci-e2e-test resourceVersion: "24197" uid: 0e74023d-7477-48e2-9e52-4e012da66728 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-08T17:04:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-cfbd9-predictor namespace: kserve-ci-e2e-test resourceVersion: "23373" uid: 74cb0607-7b29-4636-a6bf-1659f05900b5 kind: Event lastTimestamp: "2026-07-08T17:05:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-08T17:04:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-08T17:05:06Z" name: raw-sklearn-runtime-cfbd9-predictor.18c05f52b933f094 namespace: kserve-ci-e2e-test resourceVersion: "24198" uid: 81f907e4-9d62-4e0f-8743-8ce82384b672 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-08T17:03:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-cfbd9 namespace: kserve-ci-e2e-test resourceVersion: "23211" uid: d67b5538-4ff5-4452-ab81-99d408a2840a kind: Event lastTimestamp: "2026-07-08T17:04:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-08T17:03:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:04:58Z" name: raw-sklearn-runtime-cfbd9.18c05f483bc7c399 namespace: kserve-ci-e2e-test resourceVersion: "24144" uid: 2d7ad904-81db-4e19-9bd8-512e41da3c51 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:03:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-cfbd9 namespace: kserve-ci-e2e-test resourceVersion: "23211" uid: d67b5538-4ff5-4452-ab81-99d408a2840a kind: Event lastTimestamp: "2026-07-08T17:03:36Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-cfbd9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-cfbd9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-08T17:03:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:03:36Z" name: raw-sklearn-runtime-cfbd9.18c05f483c4c1d43 namespace: kserve-ci-e2e-test resourceVersion: "23225" uid: 4b262210-f0ea-49c0-acd0-874e5edbd1d1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-08T17:04:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-cfbd9 namespace: kserve-ci-e2e-test resourceVersion: "24142" uid: d67b5538-4ff5-4452-ab81-99d408a2840a kind: Event lastTimestamp: "2026-07-08T17:04:58Z" message: InferenceService [raw-sklearn-runtime-cfbd9] is Ready metadata: creationTimestamp: "2026-07-08T17:04:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-08T17:04:58Z" name: raw-sklearn-runtime-cfbd9.18c05f5b561f9c9d namespace: kserve-ci-e2e-test resourceVersion: "24143" uid: 5d7fb19a-b279-4465-9162-5fcea6629e35 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "24476"