--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:01:57.264293Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-afe32a-predictor-7f95c54965-n9tnj namespace: kserve-ci-e2e-test resourceVersion: "20904" uid: 00df6b45-2c52-4b7e-95ee-058f6e578fae kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-afe32a-predictor-7f95c54965-n9tnj to ip-10-0-143-142.ec2.internal metadata: creationTimestamp: "2026-07-09T18:01:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:01:57Z" name: isvc-init-fail-afe32a-predictor-7f95c54965-n9tnj.18c0b10befc7710b namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: 9f83197b-c04c-49ae-bdbb-75fe2cfbf153 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-74fd97cb5b-77b77 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:57Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-afe32a-predictor-7f95c54965-n9tnj namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 00df6b45-2c52-4b7e-95ee-058f6e578fae kind: Event lastTimestamp: "2026-07-09T18:01:57Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-init-fail-afe32a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T18:01:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:57Z" name: isvc-init-fail-afe32a-predictor-7f95c54965-n9tnj.18c0b10bfa60e549 namespace: kserve-ci-e2e-test resourceVersion: "20922" uid: 389db9bc-1aae-4f25-a82a-8c4db7bea9e2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-afe32a-predictor-7f95c54965-n9tnj namespace: kserve-ci-e2e-test resourceVersion: "20914" uid: 00df6b45-2c52-4b7e-95ee-058f6e578fae kind: Event lastTimestamp: "2026-07-09T18:01:58Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:01:58Z" name: isvc-init-fail-afe32a-predictor-7f95c54965-n9tnj.18c0b10c2d277252 namespace: kserve-ci-e2e-test resourceVersion: "20930" uid: af61a2c5-eacd-4971-bbe3-8e6bc02c949a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-afe32a-predictor-7f95c54965-n9tnj namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 00df6b45-2c52-4b7e-95ee-058f6e578fae kind: Event lastTimestamp: "2026-07-09T18:02:02Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6455858d9bcb13c4793d39d844d39ea45d90725b82962b5a6791b22458b6ca42" already present on machine metadata: creationTimestamp: "2026-07-09T18:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:02Z" name: isvc-init-fail-afe32a-predictor-7f95c54965-n9tnj.18c0b10c2ec91c57 namespace: kserve-ci-e2e-test resourceVersion: "20962" uid: 53b1b065-64f0-4d66-956d-cdcfbd00cd14 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-afe32a-predictor-7f95c54965-n9tnj namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 00df6b45-2c52-4b7e-95ee-058f6e578fae kind: Event lastTimestamp: "2026-07-09T18:02:02Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T18:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:02Z" name: isvc-init-fail-afe32a-predictor-7f95c54965-n9tnj.18c0b10c31d69fb2 namespace: kserve-ci-e2e-test resourceVersion: "20964" uid: 5d36f032-7213-4ef1-9355-4e7841693cd0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-afe32a-predictor-7f95c54965-n9tnj namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 00df6b45-2c52-4b7e-95ee-058f6e578fae kind: Event lastTimestamp: "2026-07-09T18:02:02Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T18:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:02Z" name: isvc-init-fail-afe32a-predictor-7f95c54965-n9tnj.18c0b10c3245e99c namespace: kserve-ci-e2e-test resourceVersion: "20965" uid: cb03ec86-0efd-4513-a457-3f78185d731a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-afe32a-predictor-7f95c54965-n9tnj namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 00df6b45-2c52-4b7e-95ee-058f6e578fae kind: Event lastTimestamp: "2026-07-09T18:02:07Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-07-09T18:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:07Z" name: isvc-init-fail-afe32a-predictor-7f95c54965-n9tnj.18c0b10e44396a7b namespace: kserve-ci-e2e-test resourceVersion: "21024" uid: 5212d341-1800-486e-8c15-d97f4016f3d8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-afe32a-predictor-7f95c54965 namespace: kserve-ci-e2e-test resourceVersion: "20897" uid: 676b5666-db45-4b34-8ed5-018a7a736fb3 kind: Event lastTimestamp: "2026-07-09T18:01:57Z" message: 'Created pod: isvc-init-fail-afe32a-predictor-7f95c54965-n9tnj' metadata: creationTimestamp: "2026-07-09T18:01:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:57Z" name: isvc-init-fail-afe32a-predictor-7f95c54965.18c0b10bef4279a9 namespace: kserve-ci-e2e-test resourceVersion: "20905" uid: c3e21657-c263-438b-8831-c4e60e639bf1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-afe32a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20895" uid: 91eaa624-7a00-480a-a00e-016ddf0a5534 kind: Event lastTimestamp: "2026-07-09T18:01:57Z" message: Scaled up replica set isvc-init-fail-afe32a-predictor-7f95c54965 from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:01:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:57Z" name: isvc-init-fail-afe32a-predictor.18c0b10becaf0b8c namespace: kserve-ci-e2e-test resourceVersion: "20898" uid: ffcfd6b9-cf29-4f38-a832-a4884cb010ff reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-09T18:01:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-afe32a namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: 3a9b7fbd-1d46-4163-8dc6-7ce7e312581e kind: Event lastTimestamp: "2026-07-09T18:02:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T18:01:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:02:02Z" name: isvc-init-fail-afe32a.18c0b10becaef0a6 namespace: kserve-ci-e2e-test resourceVersion: "20968" uid: 1d6226da-cc1f-4f68-9e45-9f9c8ec2f725 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-afe32a namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: 3a9b7fbd-1d46-4163-8dc6-7ce7e312581e kind: Event lastTimestamp: "2026-07-09T18:01:57Z" message: 'Failed to update status for InferenceService "isvc-init-fail-afe32a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-afe32a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:01:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:01:57Z" name: isvc-init-fail-afe32a.18c0b10bed1d3e80 namespace: kserve-ci-e2e-test resourceVersion: "20901" uid: 4e65342d-c831-4cca-8c34-e548cc919f77 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-afe32a namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: 3a9b7fbd-1d46-4163-8dc6-7ce7e312581e kind: Event lastTimestamp: "2026-07-09T18:01:57Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-afe32a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:01:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:01:57Z" name: isvc-init-fail-afe32a.18c0b10bed1d931f namespace: kserve-ci-e2e-test resourceVersion: "20902" uid: 87f59a4b-fd55-439e-aee0-472a7c3429a6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:55:43.112685Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17867" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf to ip-10-0-143-142.ec2.internal metadata: creationTimestamp: "2026-07-09T17:55:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:55:43Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0b4d29b0c5d namespace: kserve-ci-e2e-test resourceVersion: "17871" uid: 4e407961-862c-4a06-bf4b-3ce29d85f18d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-74fd97cb5b-77b77 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: "2026-07-09T17:55:43Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-logger-raw-1ffb5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T17:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:43Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0b4de1e9c8d namespace: kserve-ci-e2e-test resourceVersion: "17878" uid: 70a47ea4-dab2-4007-bf96-a4ef32c76ac3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:44Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17875" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: "2026-07-09T17:55:44Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:55:44Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0b50ff02bbc namespace: kserve-ci-e2e-test resourceVersion: "17887" uid: ffb5a51e-918a-4069-8762-80a7ab18b2c3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: "2026-07-09T17:55:44Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6455858d9bcb13c4793d39d844d39ea45d90725b82962b5a6791b22458b6ca42" already present on machine metadata: creationTimestamp: "2026-07-09T17:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:44Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0b51181d590 namespace: kserve-ci-e2e-test resourceVersion: "17889" uid: 1030f2be-4d70-4f34-8ae7-5a04181bc2f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: "2026-07-09T17:55:44Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T17:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:44Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0b51445f86c namespace: kserve-ci-e2e-test resourceVersion: "17892" uid: f57dee0f-76b2-4eac-b196-eee841dd138d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:44Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: "2026-07-09T17:55:44Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T17:55:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:44Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0b514b44742 namespace: kserve-ci-e2e-test resourceVersion: "17893" uid: a3768f1d-201a-460d-b20d-b46ce83c41f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: "2026-07-09T17:55:48Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T17:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:48Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0b61a7a93dd namespace: kserve-ci-e2e-test resourceVersion: "17936" uid: 81674940-7e72-4f5a-92a6-fa02e4fde2c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: "2026-07-09T17:55:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T17:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:48Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0b61ddb8396 namespace: kserve-ci-e2e-test resourceVersion: "17938" uid: 926f919a-6172-4b18-a967-d1441ce3524f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: "2026-07-09T17:55:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T17:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:48Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0b61e49106d namespace: kserve-ci-e2e-test resourceVersion: "17941" uid: 2871bfa1-fe3a-48e0-abcd-71c5e14a8c50 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: "2026-07-09T17:55:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T17:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:48Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0b61e534b4c namespace: kserve-ci-e2e-test resourceVersion: "17943" uid: 60c8a518-a754-4c8c-9607-c53e738a0c42 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: "2026-07-09T17:55:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T17:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:48Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0b62111d302 namespace: kserve-ci-e2e-test resourceVersion: "17945" uid: 5d7556f4-90bf-4f88-9650-8be8d3a18a35 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: "2026-07-09T17:55:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:48Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0b6219fe731 namespace: kserve-ci-e2e-test resourceVersion: "17946" uid: 282d2dc8-e0ad-4822-b75e-573859fcca2a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: "2026-07-09T17:55:48Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:2e623de9a93ccce79950e917affe03d05d0a7eff69143d7481fdb39990c764f1" already present on machine metadata: creationTimestamp: "2026-07-09T17:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:48Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0b621a9d586 namespace: kserve-ci-e2e-test resourceVersion: "17947" uid: 4f1e5727-8bb6-485e-b808-942e58378566 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: "2026-07-09T17:55:48Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-09T17:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:48Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0b625442a11 namespace: kserve-ci-e2e-test resourceVersion: "17948" uid: 4d77f282-26fc-405b-978b-717f2b9631b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: "2026-07-09T17:55:48Z" message: Started container agent metadata: creationTimestamp: "2026-07-09T17:55:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:48Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0b625b1bfa2 namespace: kserve-ci-e2e-test resourceVersion: "17950" uid: eee30bd1-10be-4ea9-9871-12b03d543ade reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-09T17:55:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: "2026-07-09T17:57:18Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:55:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:57:18Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0b656914fb9 namespace: kserve-ci-e2e-test resourceVersion: "18677" uid: 00048577-b721-4aa2-ba50-37185af758b9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-09T17:55:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: "2026-07-09T17:57:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T17:55:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:57:18Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0b6569ad328 namespace: kserve-ci-e2e-test resourceVersion: "18679" uid: c49fe1d9-2ba2-4ddb-89a0-913c5196cfa9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: "2026-07-09T17:57:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T17:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:57:18Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0caff064dea namespace: kserve-ci-e2e-test resourceVersion: "18633" uid: 16aee16d-7cf5-476a-8e83-cab599c7b8f0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: "2026-07-09T17:57:18Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-09T17:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:57:18Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0caff069b57 namespace: kserve-ci-e2e-test resourceVersion: "18635" uid: 5beb17ec-3b1b-4c95-a4f0-cc8da4281348 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: "2026-07-09T17:57:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:57:18Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0caff06f39c namespace: kserve-ci-e2e-test resourceVersion: "18637" uid: 7ea55b72-a4cb-4554-b84e-ac2dfbdd4534 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:57:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf namespace: kserve-ci-e2e-test resourceVersion: "17869" uid: 49aac4d7-63f6-4a9e-b578-e1e12dd043b2 kind: Event lastTimestamp: "2026-07-09T17:57:25Z" message: 'Readiness probe failed: Get "https://10.132.0.29:8643/healthz": dial tcp 10.132.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:57:25Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf.18c0b0cb86bcfcf0 namespace: kserve-ci-e2e-test resourceVersion: "18756" uid: 9de535fa-bf56-490f-8cac-4bbc438d69b7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-1ffb5-predictor-fb65b5494 namespace: kserve-ci-e2e-test resourceVersion: "17855" uid: 56a1d052-5ee6-4eed-9d18-f63bdeea2b80 kind: Event lastTimestamp: "2026-07-09T17:55:43Z" message: 'Created pod: isvc-logger-raw-1ffb5-predictor-fb65b5494-k2vgf' metadata: creationTimestamp: "2026-07-09T17:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:55:43Z" name: isvc-logger-raw-1ffb5-predictor-fb65b5494.18c0b0b4d2377409 namespace: kserve-ci-e2e-test resourceVersion: "17868" uid: 93af55a9-ca24-4422-9c0b-a06f47dfc77b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-1ffb5-predictor namespace: kserve-ci-e2e-test resourceVersion: "17853" uid: bae296b4-0437-43b4-b911-bd648f1f4712 kind: Event lastTimestamp: "2026-07-09T17:55:43Z" message: Scaled up replica set isvc-logger-raw-1ffb5-predictor-fb65b5494 from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:55:43Z" name: isvc-logger-raw-1ffb5-predictor.18c0b0b4cea492f5 namespace: kserve-ci-e2e-test resourceVersion: "17856" uid: 87ddc655-45c6-4e1f-a2ed-42d346308a49 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:55:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-1ffb5-predictor namespace: kserve-ci-e2e-test resourceVersion: "17854" uid: 6952ab8a-a810-4e6f-9477-a34cc88b6c6d kind: Event lastTimestamp: "2026-07-09T17:56:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:56:13Z" name: isvc-logger-raw-1ffb5-predictor.18c0b0b84d21664a namespace: kserve-ci-e2e-test resourceVersion: "18128" uid: 6321b3be-24d7-4899-a3fa-2b4054f868bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:55:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-1ffb5-predictor namespace: kserve-ci-e2e-test resourceVersion: "17854" uid: 6952ab8a-a810-4e6f-9477-a34cc88b6c6d kind: Event lastTimestamp: "2026-07-09T17:56:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:56:13Z" name: isvc-logger-raw-1ffb5-predictor.18c0b0b84d89e41f namespace: kserve-ci-e2e-test resourceVersion: "18129" uid: 5979abca-45ee-486d-b7b6-56297fef4b0e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T17:56:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-1ffb5-predictor namespace: kserve-ci-e2e-test resourceVersion: "18019" uid: 6952ab8a-a810-4e6f-9477-a34cc88b6c6d kind: Event lastTimestamp: "2026-07-09T17:57:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:57:13Z" name: isvc-logger-raw-1ffb5-predictor.18c0b0bf4adf0161 namespace: kserve-ci-e2e-test resourceVersion: "18568" uid: 86e8c13e-0dd9-451d-80ab-967e453af923 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T17:56:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-1ffb5-predictor namespace: kserve-ci-e2e-test resourceVersion: "18019" uid: 6952ab8a-a810-4e6f-9477-a34cc88b6c6d kind: Event lastTimestamp: "2026-07-09T17:57:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:57:13Z" name: isvc-logger-raw-1ffb5-predictor.18c0b0bf4b7f6639 namespace: kserve-ci-e2e-test resourceVersion: "18569" uid: 3f8b0909-e183-4023-9b5d-727476012a8f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-09T17:55:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-1ffb5 namespace: kserve-ci-e2e-test resourceVersion: "17846" uid: 377f42ab-f2a9-40ba-a01b-626b2c9c8759 kind: Event lastTimestamp: "2026-07-09T17:57:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T17:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:57:08Z" name: isvc-logger-raw-1ffb5.18c0b0b4cea65eff namespace: kserve-ci-e2e-test resourceVersion: "18543" uid: f9da8186-e1a9-4034-93f1-2cddd40fa7a6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-1ffb5 namespace: kserve-ci-e2e-test resourceVersion: "17846" uid: 377f42ab-f2a9-40ba-a01b-626b2c9c8759 kind: Event lastTimestamp: "2026-07-09T17:55:43Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-1ffb5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-1ffb5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T17:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:55:43Z" name: isvc-logger-raw-1ffb5.18c0b0b4cf24b503 namespace: kserve-ci-e2e-test resourceVersion: "17860" uid: 13571cb9-ec91-4dff-9008-ee6ad4544e23 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-1ffb5 namespace: kserve-ci-e2e-test resourceVersion: "18540" uid: 377f42ab-f2a9-40ba-a01b-626b2c9c8759 kind: Event lastTimestamp: "2026-07-09T17:57:08Z" message: InferenceService [isvc-logger-raw-1ffb5] is Ready metadata: creationTimestamp: "2026-07-09T17:57:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:57:08Z" name: isvc-logger-raw-1ffb5.18c0b0c8bbd56f2f namespace: kserve-ci-e2e-test resourceVersion: "18542" uid: 67f642a3-61f3-4078-b07d-3aff8c07ebdf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:00:03.761824Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-21e027-predictor-59449cf66d-qpdn5 namespace: kserve-ci-e2e-test resourceVersion: "19927" uid: 9e5eaa6d-620e-4d88-b34f-a1936c79fe1c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-21e027-predictor-59449cf66d-qpdn5 to ip-10-0-143-142.ec2.internal metadata: creationTimestamp: "2026-07-09T18:00:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:00:03Z" name: isvc-primary-21e027-predictor-59449cf66d-qpdn5.18c0b0f182813e62 namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: f5b51631-07a1-4a1f-ba70-d60b7b214d14 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-74fd97cb5b-77b77 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-21e027-predictor-59449cf66d-qpdn5 namespace: kserve-ci-e2e-test resourceVersion: "19936" uid: 9e5eaa6d-620e-4d88-b34f-a1936c79fe1c kind: Event lastTimestamp: "2026-07-09T18:00:04Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:00:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:00:04Z" name: isvc-primary-21e027-predictor-59449cf66d-qpdn5.18c0b0f19bd84d7c namespace: kserve-ci-e2e-test resourceVersion: "19942" uid: 8b62254e-5054-4d7a-82ef-082e42bd31dd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-21e027-predictor-59449cf66d-qpdn5 namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 9e5eaa6d-620e-4d88-b34f-a1936c79fe1c kind: Event lastTimestamp: "2026-07-09T18:00:04Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6455858d9bcb13c4793d39d844d39ea45d90725b82962b5a6791b22458b6ca42" already present on machine metadata: creationTimestamp: "2026-07-09T18:00:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:04Z" name: isvc-primary-21e027-predictor-59449cf66d-qpdn5.18c0b0f19d67bfeb namespace: kserve-ci-e2e-test resourceVersion: "19944" uid: 354873d0-3318-42aa-bcbf-e38612c8587c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-21e027-predictor-59449cf66d-qpdn5 namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 9e5eaa6d-620e-4d88-b34f-a1936c79fe1c kind: Event lastTimestamp: "2026-07-09T18:00:04Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T18:00:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:04Z" name: isvc-primary-21e027-predictor-59449cf66d-qpdn5.18c0b0f1a03f79d2 namespace: kserve-ci-e2e-test resourceVersion: "19945" uid: ea64a56a-dfc6-4b7c-ad53-8c7fefe1906b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:04Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-21e027-predictor-59449cf66d-qpdn5 namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 9e5eaa6d-620e-4d88-b34f-a1936c79fe1c kind: Event lastTimestamp: "2026-07-09T18:00:04Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T18:00:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:04Z" name: isvc-primary-21e027-predictor-59449cf66d-qpdn5.18c0b0f1a0ae5eac namespace: kserve-ci-e2e-test resourceVersion: "19946" uid: 94e41922-f662-475a-8ef9-7d63f69a8058 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-21e027-predictor-59449cf66d-qpdn5 namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 9e5eaa6d-620e-4d88-b34f-a1936c79fe1c kind: Event lastTimestamp: "2026-07-09T18:00:08Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T18:00:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:08Z" name: isvc-primary-21e027-predictor-59449cf66d-qpdn5.18c0b0f2a6475bc6 namespace: kserve-ci-e2e-test resourceVersion: "19986" uid: df6a6ce0-94b1-4db4-91e8-e75a332d5b55 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-21e027-predictor-59449cf66d-qpdn5 namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 9e5eaa6d-620e-4d88-b34f-a1936c79fe1c kind: Event lastTimestamp: "2026-07-09T18:00:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T18:00:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:08Z" name: isvc-primary-21e027-predictor-59449cf66d-qpdn5.18c0b0f2a9cc9376 namespace: kserve-ci-e2e-test resourceVersion: "19991" uid: 2ec247f1-9cc9-44f4-9fbb-99ac3646d28d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-21e027-predictor-59449cf66d-qpdn5 namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 9e5eaa6d-620e-4d88-b34f-a1936c79fe1c kind: Event lastTimestamp: "2026-07-09T18:00:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T18:00:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:08Z" name: isvc-primary-21e027-predictor-59449cf66d-qpdn5.18c0b0f2aa3f1dea namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: aa15c6cc-b987-40d5-87b4-741a1a1f0123 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-21e027-predictor-59449cf66d-qpdn5 namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 9e5eaa6d-620e-4d88-b34f-a1936c79fe1c kind: Event lastTimestamp: "2026-07-09T18:00:08Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T18:00:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:08Z" name: isvc-primary-21e027-predictor-59449cf66d-qpdn5.18c0b0f2aa4a1a14 namespace: kserve-ci-e2e-test resourceVersion: "19993" uid: 41364aff-c4c8-48f7-8505-129614cd5a84 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-21e027-predictor-59449cf66d-qpdn5 namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 9e5eaa6d-620e-4d88-b34f-a1936c79fe1c kind: Event lastTimestamp: "2026-07-09T18:00:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T18:00:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:08Z" name: isvc-primary-21e027-predictor-59449cf66d-qpdn5.18c0b0f2ae181a93 namespace: kserve-ci-e2e-test resourceVersion: "19997" uid: 0145ed26-b5a4-4523-bd0b-a240b943b377 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-21e027-predictor-59449cf66d-qpdn5 namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 9e5eaa6d-620e-4d88-b34f-a1936c79fe1c kind: Event lastTimestamp: "2026-07-09T18:00:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:00:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:08Z" name: isvc-primary-21e027-predictor-59449cf66d-qpdn5.18c0b0f2ae8886a4 namespace: kserve-ci-e2e-test resourceVersion: "19998" uid: 301717bc-5f1f-4650-bfe7-6efaddd1ba57 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-09T18:00:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-21e027-predictor-59449cf66d-qpdn5 namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 9e5eaa6d-620e-4d88-b34f-a1936c79fe1c kind: Event lastTimestamp: "2026-07-09T18:01:16Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:00:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:16Z" name: isvc-primary-21e027-predictor-59449cf66d-qpdn5.18c0b0f31e370367 namespace: kserve-ci-e2e-test resourceVersion: "20474" uid: 5e6eead2-87a3-4d1b-b103-8eed1b04abfe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-21e027-predictor-59449cf66d-qpdn5 namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 9e5eaa6d-620e-4d88-b34f-a1936c79fe1c kind: Event lastTimestamp: "2026-07-09T18:01:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T18:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:51Z" name: isvc-primary-21e027-predictor-59449cf66d-qpdn5.18c0b10ab5bdfe54 namespace: kserve-ci-e2e-test resourceVersion: "20818" uid: a284d910-9904-4177-8c03-0257b48d92f2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-21e027-predictor-59449cf66d-qpdn5 namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 9e5eaa6d-620e-4d88-b34f-a1936c79fe1c kind: Event lastTimestamp: "2026-07-09T18:01:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:52Z" name: isvc-primary-21e027-predictor-59449cf66d-qpdn5.18c0b10ab5be7760 namespace: kserve-ci-e2e-test resourceVersion: "20819" uid: a8b664b7-dc4d-4552-9fd3-69db54994cb0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-21e027-predictor-59449cf66d namespace: kserve-ci-e2e-test resourceVersion: "19918" uid: 07e7392a-9f9a-4e67-9b1b-dc0499c46804 kind: Event lastTimestamp: "2026-07-09T18:00:03Z" message: 'Created pod: isvc-primary-21e027-predictor-59449cf66d-qpdn5' metadata: creationTimestamp: "2026-07-09T18:00:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:00:03Z" name: isvc-primary-21e027-predictor-59449cf66d.18c0b0f1821b56f9 namespace: kserve-ci-e2e-test resourceVersion: "19928" uid: 45e7278c-aee7-4a15-b814-1fa2c9f59bcb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-21e027-predictor namespace: kserve-ci-e2e-test resourceVersion: "19916" uid: dadad315-5d9d-43f8-97ea-f0c533e4b25e kind: Event lastTimestamp: "2026-07-09T18:00:03Z" message: Scaled up replica set isvc-primary-21e027-predictor-59449cf66d from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:00:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:00:03Z" name: isvc-primary-21e027-predictor.18c0b0f17e57d102 namespace: kserve-ci-e2e-test resourceVersion: "19920" uid: 5407d4dd-358d-4bd5-8c14-e6b3bdd626cc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:00:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-21e027-predictor namespace: kserve-ci-e2e-test resourceVersion: "19917" uid: 744cc7a3-fe4e-4b9d-b918-bb40c74a81e9 kind: Event lastTimestamp: "2026-07-09T18:00:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:00:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:00:48Z" name: isvc-primary-21e027-predictor.18c0b0f4fd39fcea namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: fba4a94c-4572-406d-b811-3c07c84b8d5c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:00:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-21e027-predictor namespace: kserve-ci-e2e-test resourceVersion: "19917" uid: 744cc7a3-fe4e-4b9d-b918-bb40c74a81e9 kind: Event lastTimestamp: "2026-07-09T18:00:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:00:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:00:48Z" name: isvc-primary-21e027-predictor.18c0b0f4fd9f3143 namespace: kserve-ci-e2e-test resourceVersion: "20283" uid: 0ca58b2e-1b39-4241-9cfd-72da919ddb39 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T18:00:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-21e027-predictor namespace: kserve-ci-e2e-test resourceVersion: "20068" uid: 744cc7a3-fe4e-4b9d-b918-bb40c74a81e9 kind: Event lastTimestamp: "2026-07-09T18:01:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:00:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:48Z" name: isvc-primary-21e027-predictor.18c0b0f87cc300b5 namespace: kserve-ci-e2e-test resourceVersion: "20774" uid: 1e239e07-ad60-4d1e-af29-9ff39a593a60 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-09T18:00:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-21e027-predictor namespace: kserve-ci-e2e-test resourceVersion: "20068" uid: 744cc7a3-fe4e-4b9d-b918-bb40c74a81e9 kind: Event lastTimestamp: "2026-07-09T18:01:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:00:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:48Z" name: isvc-primary-21e027-predictor.18c0b0f87d0b8496 namespace: kserve-ci-e2e-test resourceVersion: "20775" uid: 36588c3a-edb5-4a00-a3c0-f03c7a0fc3bf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-09T18:00:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-21e027 namespace: kserve-ci-e2e-test resourceVersion: "19909" uid: db26f6c4-a679-4440-b4c3-d57957266a78 kind: Event lastTimestamp: "2026-07-09T18:01:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T18:00:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:01:26Z" name: isvc-primary-21e027.18c0b0f17e4f79d0 namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: a6af7613-6be4-4fd6-88c7-0a6c15a00761 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-21e027 namespace: kserve-ci-e2e-test resourceVersion: "19909" uid: db26f6c4-a679-4440-b4c3-d57957266a78 kind: Event lastTimestamp: "2026-07-09T18:00:03Z" message: 'Failed to update status for InferenceService "isvc-primary-21e027": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-21e027": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:00:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:00:03Z" name: isvc-primary-21e027.18c0b0f17e9bdeba namespace: kserve-ci-e2e-test resourceVersion: "19922" uid: b3a1ac2c-9d70-4e36-acae-1aae0f9b5a83 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:00:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-21e027 namespace: kserve-ci-e2e-test resourceVersion: "19909" uid: db26f6c4-a679-4440-b4c3-d57957266a78 kind: Event lastTimestamp: "2026-07-09T18:00:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-21e027": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:00:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:00:03Z" name: isvc-primary-21e027.18c0b0f17e9c422a namespace: kserve-ci-e2e-test resourceVersion: "19923" uid: 3a1d7e7e-6bb5-48a7-8b15-e1e1a52a1c3f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-21e027 namespace: kserve-ci-e2e-test resourceVersion: "20554" uid: db26f6c4-a679-4440-b4c3-d57957266a78 kind: Event lastTimestamp: "2026-07-09T18:01:26Z" message: InferenceService [isvc-primary-21e027] is Ready metadata: creationTimestamp: "2026-07-09T18:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:01:26Z" name: isvc-primary-21e027.18c0b104d761e80b namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: dd44008e-9f6a-4333-be00-e28faf814bb2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:49:47.180748Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14661" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 to ip-10-0-143-142.ec2.internal metadata: creationTimestamp: "2026-07-09T17:49:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:49:47Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b061f368fd4a namespace: kserve-ci-e2e-test resourceVersion: "14670" uid: 82227c93-47a1-47ec-9fc5-bcedfe1a8deb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-74fd97cb5b-77b77 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:47Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14672" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:49:47Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:49:47Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b0620cb8315d namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: c6b47015-81be-46a8-9543-2830b083647c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:49:47Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6455858d9bcb13c4793d39d844d39ea45d90725b82962b5a6791b22458b6ca42" already present on machine metadata: creationTimestamp: "2026-07-09T17:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:47Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b0620e74f9d1 namespace: kserve-ci-e2e-test resourceVersion: "14682" uid: 0989c02b-42fa-4fb3-8ac8-e24e1f86dec9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:49:47Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T17:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:47Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b06211364fb5 namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: b297ae7b-af0b-4b38-b5ee-5b54a5cfb3c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:49:47Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T17:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:47Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b06211a6c5fd namespace: kserve-ci-e2e-test resourceVersion: "14685" uid: 082746fb-a9ac-4cb4-a67a-f9022eacd8f3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:49:52Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1714" metadata: creationTimestamp: "2026-07-09T17:49:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:49:52Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b063190ba0b3 namespace: kserve-ci-e2e-test resourceVersion: "14718" uid: dcfee00b-e13c-4064-bd10-1872499a9ccc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:50:25Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1714" in 33.582s (33.582s including waiting). Image size: 1555846059 bytes.' metadata: creationTimestamp: "2026-07-09T17:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:25Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b06aeab48013 namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: c7f01b51-30e3-4f64-964c-f0db98c07da5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:50:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T17:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:25Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b06af14e90a9 namespace: kserve-ci-e2e-test resourceVersion: "14955" uid: 29b8d582-9fd2-489a-aa74-10eee6087dc9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:50:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T17:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:25Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b06af2747e1b namespace: kserve-ci-e2e-test resourceVersion: "14956" uid: bf2558e9-3b8e-42ed-8e1e-c24dea3deec0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:50:25Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-09T17:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:25Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b06af281143c namespace: kserve-ci-e2e-test resourceVersion: "14957" uid: 077a843b-4f61-4d54-ad2f-358dfe46cb9c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:50:28Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.402s (2.402s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-09T17:50:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:28Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b06b81b097f1 namespace: kserve-ci-e2e-test resourceVersion: "14969" uid: a0335cc5-dc58-4e81-8010-4188f8392183 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:50:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T17:50:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:28Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b06b85a1c010 namespace: kserve-ci-e2e-test resourceVersion: "14970" uid: 1f10b145-42d8-4448-a0db-cba9c10510e5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:50:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:50:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:28Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b06b8612bb79 namespace: kserve-ci-e2e-test resourceVersion: "14972" uid: 61bacc63-33b5-4bf4-9f93-4268a19f0a6a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:50:28Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:2e623de9a93ccce79950e917affe03d05d0a7eff69143d7481fdb39990c764f1" metadata: creationTimestamp: "2026-07-09T17:50:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:28Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b06b861ee88a namespace: kserve-ci-e2e-test resourceVersion: "14973" uid: a04318c4-ece4-45cc-84ca-1e1605742a68 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:50:30Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:2e623de9a93ccce79950e917affe03d05d0a7eff69143d7481fdb39990c764f1" in 2.418s (2.418s including waiting). Image size: 239997026 bytes.' metadata: creationTimestamp: "2026-07-09T17:50:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:30Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b06c16432a6b namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: a6788251-6f64-4339-ad07-2dce4c4f9158 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:50:30Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-09T17:50:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:30Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b06c197ee454 namespace: kserve-ci-e2e-test resourceVersion: "14994" uid: 75ed08ac-8920-4d9e-9f93-a6db881c9de8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:50:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:50:30Z" message: Started container agent metadata: creationTimestamp: "2026-07-09T17:50:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:50:30Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b06c19f3942a namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: f03eba2a-4765-457d-9e8a-e9fb194a502f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-09T17:50:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:52:04Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:50:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:52:04Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b06c74151f22 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: 2ecbf945-ed5c-4d1a-b064-cadd3c6dc061 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-09T17:50:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:52:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T17:50:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:52:04Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b06c741f89d5 namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: ef4239f3-e0da-4a28-977c-89d7f015fdd1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:51:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T17:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:52Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b07f11922bdc namespace: kserve-ci-e2e-test resourceVersion: "15620" uid: 309f118d-9747-4df8-ae6b-f31bd605c4e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:51:52Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-09T17:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:52Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b07f1192601b namespace: kserve-ci-e2e-test resourceVersion: "15621" uid: 35d7caa8-0d6a-4d0e-b575-d7bf75024666 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:51:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:52Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b07f1192d028 namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: 51a577e5-a87c-43c4-b9bd-ebe73e571cda reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:51:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: e19d7cea-61c3-4e82-a6f7-10b7b8fd864f kind: Event lastTimestamp: "2026-07-09T17:52:03Z" message: 'Readiness probe failed: Get "https://10.132.0.23:8643/healthz": dial tcp 10.132.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:52:03Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9.18c0b07f504779a6 namespace: kserve-ci-e2e-test resourceVersion: "15807" uid: 086d5379-20df-4536-aa63-c7a1d9795464 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c namespace: kserve-ci-e2e-test resourceVersion: "14646" uid: 78445476-69ce-4b26-b136-245237442ffe kind: Event lastTimestamp: "2026-07-09T17:49:47Z" message: 'Created pod: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c-49cd9' metadata: creationTimestamp: "2026-07-09T17:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:49:47Z" name: isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c.18c0b061f28a0212 namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: f3c0ac6c-44d1-4ba0-a3cb-d2ff81a7ab39 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-fc881-predictor namespace: kserve-ci-e2e-test resourceVersion: "14643" uid: a1afb166-728f-4faa-a4c5-c5ae55208266 kind: Event lastTimestamp: "2026-07-09T17:49:46Z" message: Scaled up replica set isvc-raw-sklearn-batcher-fc881-predictor-78dc4575c from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:49:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:49:46Z" name: isvc-raw-sklearn-batcher-fc881-predictor.18c0b061e86e8ec4 namespace: kserve-ci-e2e-test resourceVersion: "14647" uid: a32f6f1b-bedf-4781-878b-eaaee4a3ee7e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T17:50:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-fc881-predictor namespace: kserve-ci-e2e-test resourceVersion: "14655" uid: 3352f754-0bf5-43bc-928a-1409cf586ca2 kind: Event lastTimestamp: "2026-07-09T17:51:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:50:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:02Z" name: isvc-raw-sklearn-batcher-fc881-predictor.18c0b0656ea4308e namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: fe5d76ad-17f1-480b-937b-705714535935 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T17:50:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-fc881-predictor namespace: kserve-ci-e2e-test resourceVersion: "14655" uid: 3352f754-0bf5-43bc-928a-1409cf586ca2 kind: Event lastTimestamp: "2026-07-09T17:51:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:50:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:02Z" name: isvc-raw-sklearn-batcher-fc881-predictor.18c0b0656f1be15d namespace: kserve-ci-e2e-test resourceVersion: "15239" uid: 9e950fcc-58bd-4afc-9ed4-cb9b88d96026 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T17:50:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-fc881-predictor namespace: kserve-ci-e2e-test resourceVersion: "14789" uid: 3352f754-0bf5-43bc-928a-1409cf586ca2 kind: Event lastTimestamp: "2026-07-09T17:51:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:50:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:47Z" name: isvc-raw-sklearn-batcher-fc881-predictor.18c0b06fee9337a4 namespace: kserve-ci-e2e-test resourceVersion: "15558" uid: 2311b520-d1c1-4366-bc46-7764497b03fc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T17:50:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-fc881-predictor namespace: kserve-ci-e2e-test resourceVersion: "14789" uid: 3352f754-0bf5-43bc-928a-1409cf586ca2 kind: Event lastTimestamp: "2026-07-09T17:51:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:50:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:47Z" name: isvc-raw-sklearn-batcher-fc881-predictor.18c0b06fef0580fc namespace: kserve-ci-e2e-test resourceVersion: "15559" uid: d62b7438-7b30-4ff1-81b2-33c95e836d85 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-09T17:49:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-fc881 namespace: kserve-ci-e2e-test resourceVersion: "14634" uid: 3b7a30bf-5e0c-43b5-916e-b5aebf19e824 kind: Event lastTimestamp: "2026-07-09T17:51:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T17:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:51:44Z" name: isvc-raw-sklearn-batcher-fc881.18c0b061eeddd31d namespace: kserve-ci-e2e-test resourceVersion: "15542" uid: 4b6530c1-f448-4690-a764-fcfeef3936f0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:49:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-fc881 namespace: kserve-ci-e2e-test resourceVersion: "14634" uid: 3b7a30bf-5e0c-43b5-916e-b5aebf19e824 kind: Event lastTimestamp: "2026-07-09T17:49:47Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-fc881": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-fc881": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T17:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:49:47Z" name: isvc-raw-sklearn-batcher-fc881.18c0b061ef364c7a namespace: kserve-ci-e2e-test resourceVersion: "14674" uid: 68eba27b-3958-4309-8141-2ac8d6fad9b2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:49:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-fc881 namespace: kserve-ci-e2e-test resourceVersion: "14638" uid: 3b7a30bf-5e0c-43b5-916e-b5aebf19e824 kind: Event lastTimestamp: "2026-07-09T17:49:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-fc881": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T17:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:49:47Z" name: isvc-raw-sklearn-batcher-fc881.18c0b061f85b9d56 namespace: kserve-ci-e2e-test resourceVersion: "14675" uid: 983c3c89-69e4-4bbf-9a19-1c37506c3347 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-fc881 namespace: kserve-ci-e2e-test resourceVersion: "15539" uid: 3b7a30bf-5e0c-43b5-916e-b5aebf19e824 kind: Event lastTimestamp: "2026-07-09T17:51:44Z" message: InferenceService [isvc-raw-sklearn-batcher-fc881] is Ready metadata: creationTimestamp: "2026-07-09T17:51:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:51:44Z" name: isvc-raw-sklearn-batcher-fc881.18c0b07d3f268e2d namespace: kserve-ci-e2e-test resourceVersion: "15541" uid: 5a6dd0b6-07e6-4d23-9a6c-b0a60c281122 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:01:33.971969Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-21e027-predictor-6d849b584c-p4tz8 namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: d3536bec-6d76-4d3e-ae0c-9d38806500f8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-21e027-predictor-6d849b584c-p4tz8 to ip-10-0-143-142.ec2.internal metadata: creationTimestamp: "2026-07-09T18:01:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:01:33Z" name: isvc-secondary-21e027-predictor-6d849b584c-p4tz8.18c0b1068372d7c2 namespace: kserve-ci-e2e-test resourceVersion: "20639" uid: a66f7684-3a60-4947-b485-6c206df8da67 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-74fd97cb5b-77b77 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:34Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-21e027-predictor-6d849b584c-p4tz8 namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: d3536bec-6d76-4d3e-ae0c-9d38806500f8 kind: Event lastTimestamp: "2026-07-09T18:01:34Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-secondary-21e027-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T18:01:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:34Z" name: isvc-secondary-21e027-predictor-6d849b584c-p4tz8.18c0b1068e97d394 namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: f7a19642-e7f6-4a87-bbd9-26ce65853a6e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:34Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-21e027-predictor-6d849b584c-p4tz8 namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: d3536bec-6d76-4d3e-ae0c-9d38806500f8 kind: Event lastTimestamp: "2026-07-09T18:01:34Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:01:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:01:34Z" name: isvc-secondary-21e027-predictor-6d849b584c-p4tz8.18c0b106c09139d8 namespace: kserve-ci-e2e-test resourceVersion: "20652" uid: ed73e0cb-b7d0-4bf0-b99e-2d26d88da6b8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:01:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-21e027-predictor-6d849b584c-p4tz8 namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: d3536bec-6d76-4d3e-ae0c-9d38806500f8 kind: Event lastTimestamp: "2026-07-09T18:01:41Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6455858d9bcb13c4793d39d844d39ea45d90725b82962b5a6791b22458b6ca42" already present on machine metadata: creationTimestamp: "2026-07-09T18:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:41Z" name: isvc-secondary-21e027-predictor-6d849b584c-p4tz8.18c0b106c2289543 namespace: kserve-ci-e2e-test resourceVersion: "20708" uid: 9b447be9-c1fb-494c-bca6-5adf39c69dcd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:01:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-21e027-predictor-6d849b584c-p4tz8 namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: d3536bec-6d76-4d3e-ae0c-9d38806500f8 kind: Event lastTimestamp: "2026-07-09T18:01:41Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T18:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:41Z" name: isvc-secondary-21e027-predictor-6d849b584c-p4tz8.18c0b106c53bc594 namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: d94b402d-3ff4-425f-94d8-b9594b4be2ea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:01:35Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-21e027-predictor-6d849b584c-p4tz8 namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: d3536bec-6d76-4d3e-ae0c-9d38806500f8 kind: Event lastTimestamp: "2026-07-09T18:01:41Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T18:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:41Z" name: isvc-secondary-21e027-predictor-6d849b584c-p4tz8.18c0b106c5acfa02 namespace: kserve-ci-e2e-test resourceVersion: "20713" uid: d582713a-842c-42aa-84ef-bc4428ee4a7b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-21e027-predictor-6d849b584c-p4tz8 namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: d3536bec-6d76-4d3e-ae0c-9d38806500f8 kind: Event lastTimestamp: "2026-07-09T18:01:47Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-21e027-predictor-6d849b584c-p4tz8_kserve-ci-e2e-test(d3536bec-6d76-4d3e-ae0c-9d38806500f8) metadata: creationTimestamp: "2026-07-09T18:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:01:47Z" name: isvc-secondary-21e027-predictor-6d849b584c-p4tz8.18c0b1098f914bb6 namespace: kserve-ci-e2e-test resourceVersion: "20756" uid: d630944a-6535-4843-8eae-df0d4b65b720 reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-21e027-predictor-6d849b584c namespace: kserve-ci-e2e-test resourceVersion: "20625" uid: 2bf0e91f-b361-43f7-8654-860ee0ec74b2 kind: Event lastTimestamp: "2026-07-09T18:01:33Z" message: 'Created pod: isvc-secondary-21e027-predictor-6d849b584c-p4tz8' metadata: creationTimestamp: "2026-07-09T18:01:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:33Z" name: isvc-secondary-21e027-predictor-6d849b584c.18c0b10683182474 namespace: kserve-ci-e2e-test resourceVersion: "20637" uid: c3d2adc4-852c-4477-8cee-4bfa77cf9769 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-21e027-predictor namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: 8f89aaf7-cecc-4435-bab3-d47f48bf2519 kind: Event lastTimestamp: "2026-07-09T18:01:33Z" message: Scaled up replica set isvc-secondary-21e027-predictor-6d849b584c from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:01:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:33Z" name: isvc-secondary-21e027-predictor.18c0b1067edef4a9 namespace: kserve-ci-e2e-test resourceVersion: "20627" uid: bc565973-7375-4f8a-8e8a-2253ddadb016 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-21e027-predictor namespace: kserve-ci-e2e-test resourceVersion: "20624" uid: 308387f5-0952-4643-93ca-1b46662db857 kind: Event lastTimestamp: "2026-07-09T18:01:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:48Z" name: isvc-secondary-21e027-predictor.18c0b109fd6bac51 namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: ea56640e-bb71-4c9e-b503-01b132cf3b68 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-21e027-predictor namespace: kserve-ci-e2e-test resourceVersion: "20624" uid: 308387f5-0952-4643-93ca-1b46662db857 kind: Event lastTimestamp: "2026-07-09T18:01:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:01:48Z" name: isvc-secondary-21e027-predictor.18c0b109fddd5150 namespace: kserve-ci-e2e-test resourceVersion: "20778" uid: 7798a0a9-00c9-4953-b031-b474a2b9e573 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-09T18:01:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-21e027 namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 5852e0f1-e828-4988-8734-ce02a5a38191 kind: Event lastTimestamp: "2026-07-09T18:01:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T18:01:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:01:47Z" name: isvc-secondary-21e027.18c0b1067f0db0de namespace: kserve-ci-e2e-test resourceVersion: "20762" uid: 058036bc-84ea-4fd2-b820-5f36719c3adb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-21e027 namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 5852e0f1-e828-4988-8734-ce02a5a38191 kind: Event lastTimestamp: "2026-07-09T18:01:33Z" message: 'Failed to update status for InferenceService "isvc-secondary-21e027": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-21e027": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:01:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:01:33Z" name: isvc-secondary-21e027.18c0b1067f68d11f namespace: kserve-ci-e2e-test resourceVersion: "20630" uid: 15a9ce12-9a16-4ced-b6ba-170ff01ba83b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:01:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-21e027 namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 5852e0f1-e828-4988-8734-ce02a5a38191 kind: Event lastTimestamp: "2026-07-09T18:01:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-21e027": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:01:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:01:33Z" name: isvc-secondary-21e027.18c0b1067f6919d5 namespace: kserve-ci-e2e-test resourceVersion: "20631" uid: 031ab0ab-3b34-49bc-94f9-8d49eaed06e9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:51:52.389944Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd namespace: kserve-ci-e2e-test resourceVersion: "15645" uid: b47d39ca-aae9-4e35-b703-57e80dcaead3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd to ip-10-0-143-142.ec2.internal metadata: creationTimestamp: "2026-07-09T17:51:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:51:52Z" name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd.18c0b07f1a75a697 namespace: kserve-ci-e2e-test resourceVersion: "15656" uid: e735845a-28f3-406f-aeeb-fb4511ecb728 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-74fd97cb5b-77b77 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd namespace: kserve-ci-e2e-test resourceVersion: "15668" uid: b47d39ca-aae9-4e35-b703-57e80dcaead3 kind: Event lastTimestamp: "2026-07-09T17:51:52Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:51:52Z" name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd.18c0b07f34517b14 namespace: kserve-ci-e2e-test resourceVersion: "15694" uid: 54d32ad2-38de-48eb-902c-4146c395c701 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: b47d39ca-aae9-4e35-b703-57e80dcaead3 kind: Event lastTimestamp: "2026-07-09T17:51:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6455858d9bcb13c4793d39d844d39ea45d90725b82962b5a6791b22458b6ca42" already present on machine metadata: creationTimestamp: "2026-07-09T17:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:52Z" name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd.18c0b07f35f6c00c namespace: kserve-ci-e2e-test resourceVersion: "15696" uid: bc529825-a80c-47d5-a5f1-7c07e54348dd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: b47d39ca-aae9-4e35-b703-57e80dcaead3 kind: Event lastTimestamp: "2026-07-09T17:51:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T17:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:52Z" name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd.18c0b07f3915cb77 namespace: kserve-ci-e2e-test resourceVersion: "15697" uid: 808ad917-3a05-4283-8779-038618a3d10c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: b47d39ca-aae9-4e35-b703-57e80dcaead3 kind: Event lastTimestamp: "2026-07-09T17:51:52Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T17:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:52Z" name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd.18c0b07f3991c032 namespace: kserve-ci-e2e-test resourceVersion: "15699" uid: bc3e9eda-7cc4-4fa5-b050-a6bd04d70310 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: b47d39ca-aae9-4e35-b703-57e80dcaead3 kind: Event lastTimestamp: "2026-07-09T17:51:56Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T17:51:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:56Z" name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd.18c0b08016acecdd namespace: kserve-ci-e2e-test resourceVersion: "15748" uid: f2576669-eede-47ab-80cc-bd3d4db8afe7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: b47d39ca-aae9-4e35-b703-57e80dcaead3 kind: Event lastTimestamp: "2026-07-09T17:51:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T17:51:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:56Z" name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd.18c0b0801c8e3dff namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: 3de9375b-a85d-4b41-84e2-2315a792aa17 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: b47d39ca-aae9-4e35-b703-57e80dcaead3 kind: Event lastTimestamp: "2026-07-09T17:51:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T17:51:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:56Z" name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd.18c0b0801d0129d0 namespace: kserve-ci-e2e-test resourceVersion: "15755" uid: e6e4fce1-3ec1-42ff-a4f4-96ccc08c0c1d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: b47d39ca-aae9-4e35-b703-57e80dcaead3 kind: Event lastTimestamp: "2026-07-09T17:51:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T17:51:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:56Z" name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd.18c0b0801d0d2f51 namespace: kserve-ci-e2e-test resourceVersion: "15756" uid: 38f967c9-13f5-4280-a374-63a7667dd31c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: b47d39ca-aae9-4e35-b703-57e80dcaead3 kind: Event lastTimestamp: "2026-07-09T17:51:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T17:51:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:56Z" name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd.18c0b0801fddc9ac namespace: kserve-ci-e2e-test resourceVersion: "15757" uid: 2206bcf1-d9b9-4bc4-887e-5e69c2e92acd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: b47d39ca-aae9-4e35-b703-57e80dcaead3 kind: Event lastTimestamp: "2026-07-09T17:51:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:51:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:56Z" name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd.18c0b0802052c521 namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: 39324516-dfac-451c-86ec-615b9870a0d3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-09T17:51:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: b47d39ca-aae9-4e35-b703-57e80dcaead3 kind: Event lastTimestamp: "2026-07-09T17:53:43Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:51:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:43Z" name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd.18c0b08052daf081 namespace: kserve-ci-e2e-test resourceVersion: "16743" uid: 5dd9f6ae-6817-4348-9edb-dc00b67b24ef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: b47d39ca-aae9-4e35-b703-57e80dcaead3 kind: Event lastTimestamp: "2026-07-09T17:53:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T17:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:42Z" name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd.18c0b098c68123dc namespace: kserve-ci-e2e-test resourceVersion: "16657" uid: 3e517b8f-a179-4026-9ac5-aaefc49cb39e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: b47d39ca-aae9-4e35-b703-57e80dcaead3 kind: Event lastTimestamp: "2026-07-09T17:53:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:42Z" name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd.18c0b098c6816bf9 namespace: kserve-ci-e2e-test resourceVersion: "16662" uid: 590302d3-1b12-4e24-9257-ab10b817f322 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd namespace: kserve-ci-e2e-test resourceVersion: "15650" uid: b47d39ca-aae9-4e35-b703-57e80dcaead3 kind: Event lastTimestamp: "2026-07-09T17:53:43Z" message: 'Readiness probe failed: Get "https://10.132.0.24:8643/healthz": dial tcp 10.132.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:43Z" name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd.18c0b09901307ef1 namespace: kserve-ci-e2e-test resourceVersion: "16742" uid: 286608e4-86f1-4c39-9295-56456416732f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465 namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: 3e43c3f0-c9ce-41f0-a759-37705731e331 kind: Event lastTimestamp: "2026-07-09T17:51:52Z" message: 'Created pod: isvc-sklearn-graph-raw-02e34-predictor-78ff766465-b5kkd' metadata: creationTimestamp: "2026-07-09T17:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:52Z" name: isvc-sklearn-graph-raw-02e34-predictor-78ff766465.18c0b07f19fc8ae3 namespace: kserve-ci-e2e-test resourceVersion: "15649" uid: 7f7aab3a-8128-420a-a60e-71ee5def7ea1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-02e34-predictor namespace: kserve-ci-e2e-test resourceVersion: "15631" uid: 61c9241e-3782-4840-8fe6-b4978b615852 kind: Event lastTimestamp: "2026-07-09T17:51:52Z" message: Scaled up replica set isvc-sklearn-graph-raw-02e34-predictor-78ff766465 from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:52Z" name: isvc-sklearn-graph-raw-02e34-predictor.18c0b07f152848f1 namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: e35fa57b-ab86-453f-9453-50529a9eb1f5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:52:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-02e34-predictor namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: 46579499-40b6-406d-a16a-abc719225116 kind: Event lastTimestamp: "2026-07-09T17:52:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:52:37Z" name: isvc-sklearn-graph-raw-02e34-predictor.18c0b08294bf0964 namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: 5191c43c-8ec1-44d8-8630-ef9337d16e0b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:52:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-02e34-predictor namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: 46579499-40b6-406d-a16a-abc719225116 kind: Event lastTimestamp: "2026-07-09T17:52:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:52:37Z" name: isvc-sklearn-graph-raw-02e34-predictor.18c0b08295642d74 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: f4bbb8c0-8c01-4048-bdeb-61fad88b7a4f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T17:52:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-02e34-predictor namespace: kserve-ci-e2e-test resourceVersion: "15842" uid: 46579499-40b6-406d-a16a-abc719225116 kind: Event lastTimestamp: "2026-07-09T17:53:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:53:37Z" name: isvc-sklearn-graph-raw-02e34-predictor.18c0b08d122a2c1c namespace: kserve-ci-e2e-test resourceVersion: "16563" uid: e48f238f-a89f-41aa-ad50-621c31c7a159 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T17:52:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-02e34-predictor namespace: kserve-ci-e2e-test resourceVersion: "15842" uid: 46579499-40b6-406d-a16a-abc719225116 kind: Event lastTimestamp: "2026-07-09T17:53:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:53:37Z" name: isvc-sklearn-graph-raw-02e34-predictor.18c0b08d12a58327 namespace: kserve-ci-e2e-test resourceVersion: "16564" uid: 304068f4-b116-4bd5-98b9-d0f7a5c71a50 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-09T17:51:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-02e34 namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: 7aa35c02-c328-408b-a802-c6cd772b44b1 kind: Event lastTimestamp: "2026-07-09T17:53:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T17:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:53:13Z" name: isvc-sklearn-graph-raw-02e34.18c0b07f15533101 namespace: kserve-ci-e2e-test resourceVersion: "16352" uid: d8886179-e1e4-4165-97d6-c6e1689bd260 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-02e34 namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: 7aa35c02-c328-408b-a802-c6cd772b44b1 kind: Event lastTimestamp: "2026-07-09T17:53:13Z" message: InferenceService [isvc-sklearn-graph-raw-02e34] is Ready metadata: creationTimestamp: "2026-07-09T17:53:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:53:13Z" name: isvc-sklearn-graph-raw-02e34.18c0b0920bed8bdf namespace: kserve-ci-e2e-test resourceVersion: "16351" uid: 080d3069-2aea-4676-9631-9e7cd28edc87 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:53:42.718469Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: add774cf-4d7d-4e0a-a86f-422fa5c16871 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf to ip-10-0-143-142.ec2.internal metadata: creationTimestamp: "2026-07-09T17:53:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:53:42Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf.18c0b098ca8d25ee namespace: kserve-ci-e2e-test resourceVersion: "16678" uid: f7533e68-e3e1-49dd-89fa-c9caf586e530 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-74fd97cb5b-77b77 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf namespace: kserve-ci-e2e-test resourceVersion: "16694" uid: add774cf-4d7d-4e0a-a86f-422fa5c16871 kind: Event lastTimestamp: "2026-07-09T17:53:43Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:53:43Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf.18c0b098e454d1e7 namespace: kserve-ci-e2e-test resourceVersion: "16734" uid: 3160563a-774e-4028-a1f5-3d5a4bf29b5c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: add774cf-4d7d-4e0a-a86f-422fa5c16871 kind: Event lastTimestamp: "2026-07-09T17:53:43Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6455858d9bcb13c4793d39d844d39ea45d90725b82962b5a6791b22458b6ca42" already present on machine metadata: creationTimestamp: "2026-07-09T17:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:43Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf.18c0b098e5fa88a3 namespace: kserve-ci-e2e-test resourceVersion: "16737" uid: 33c616d5-691f-4354-bc96-6bb752566b05 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: add774cf-4d7d-4e0a-a86f-422fa5c16871 kind: Event lastTimestamp: "2026-07-09T17:53:43Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T17:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:43Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf.18c0b098e8e67bb6 namespace: kserve-ci-e2e-test resourceVersion: "16738" uid: aab88697-fa6a-477b-9cb8-b86316cdfb95 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: add774cf-4d7d-4e0a-a86f-422fa5c16871 kind: Event lastTimestamp: "2026-07-09T17:53:43Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T17:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:43Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf.18c0b098e9599560 namespace: kserve-ci-e2e-test resourceVersion: "16739" uid: f6c76c8a-2951-4de5-9f27-edc2f3996aea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: add774cf-4d7d-4e0a-a86f-422fa5c16871 kind: Event lastTimestamp: "2026-07-09T17:53:47Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T17:53:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:47Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf.18c0b099cf394ed1 namespace: kserve-ci-e2e-test resourceVersion: "16786" uid: de372e63-ae79-4a8e-99a4-f58bc143f3e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: add774cf-4d7d-4e0a-a86f-422fa5c16871 kind: Event lastTimestamp: "2026-07-09T17:53:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T17:53:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:47Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf.18c0b099d35d98c0 namespace: kserve-ci-e2e-test resourceVersion: "16790" uid: 48d4645f-e5df-4808-a3b5-e60b2baee49d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: add774cf-4d7d-4e0a-a86f-422fa5c16871 kind: Event lastTimestamp: "2026-07-09T17:53:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T17:53:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:47Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf.18c0b099d3d29b3a namespace: kserve-ci-e2e-test resourceVersion: "16791" uid: af8504be-820e-4ff2-bb37-8bc3cd2137bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: add774cf-4d7d-4e0a-a86f-422fa5c16871 kind: Event lastTimestamp: "2026-07-09T17:53:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T17:53:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:47Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf.18c0b099d3ddeefb namespace: kserve-ci-e2e-test resourceVersion: "16792" uid: d96bac2f-b25c-45f6-a4dc-00bd7ee9d392 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: add774cf-4d7d-4e0a-a86f-422fa5c16871 kind: Event lastTimestamp: "2026-07-09T17:53:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T17:53:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:47Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf.18c0b099d67d96a8 namespace: kserve-ci-e2e-test resourceVersion: "16795" uid: 5a54a74e-0877-440b-8d88-a7729b4503f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: add774cf-4d7d-4e0a-a86f-422fa5c16871 kind: Event lastTimestamp: "2026-07-09T17:53:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:53:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:47Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf.18c0b099d6ef70bf namespace: kserve-ci-e2e-test resourceVersion: "16796" uid: 3946b4b2-1098-4e0f-b444-61314108c4dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-09T17:53:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: add774cf-4d7d-4e0a-a86f-422fa5c16871 kind: Event lastTimestamp: "2026-07-09T17:54:55Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:53:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:54:55Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf.18c0b09a47ac897f namespace: kserve-ci-e2e-test resourceVersion: "17316" uid: 35d85767-c336-48eb-aa0e-6aba1734bae3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: add774cf-4d7d-4e0a-a86f-422fa5c16871 kind: Event lastTimestamp: "2026-07-09T17:55:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T17:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:23Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf.18c0b0b025750d5a namespace: kserve-ci-e2e-test resourceVersion: "17625" uid: aa952f48-47c3-4c31-a619-baf6e6cda8bb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: add774cf-4d7d-4e0a-a86f-422fa5c16871 kind: Event lastTimestamp: "2026-07-09T17:55:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:23Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf.18c0b0b02575f088 namespace: kserve-ci-e2e-test resourceVersion: "17627" uid: e7427c31-4637-477e-9619-604dff2bdb66 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: add774cf-4d7d-4e0a-a86f-422fa5c16871 kind: Event lastTimestamp: "2026-07-09T17:55:25Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:25Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf.18c0b0b0a20b3d20 namespace: kserve-ci-e2e-test resourceVersion: "17692" uid: 006d0443-8117-4588-862a-7dec1084c9b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849 namespace: kserve-ci-e2e-test resourceVersion: "16645" uid: 7e5c0896-62e9-43d8-93ae-540a2e087808 kind: Event lastTimestamp: "2026-07-09T17:53:42Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849-8f7bf' metadata: creationTimestamp: "2026-07-09T17:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:53:42Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849.18c0b098ca33a7f8 namespace: kserve-ci-e2e-test resourceVersion: "16673" uid: 0794c5e5-61b5-49eb-8e13-9331d40598ea reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-079fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: 5ad0ec4c-c531-404d-a4ac-6d6803d2f22f kind: Event lastTimestamp: "2026-07-09T17:53:42Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-079fb-predictor-74f5878849 from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:53:42Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor.18c0b098c577a423 namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: c0f72f3f-9e0f-40f3-b00e-5f654af02767 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:53:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-079fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16642" uid: 13f2c16f-d808-4991-bf33-6804010560fa kind: Event lastTimestamp: "2026-07-09T17:54:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:54:12Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor.18c0b09c43a49d18 namespace: kserve-ci-e2e-test resourceVersion: "17004" uid: 2474d738-9024-44cb-bd17-10e8be9e82cb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:53:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-079fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16642" uid: 13f2c16f-d808-4991-bf33-6804010560fa kind: Event lastTimestamp: "2026-07-09T17:54:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:54:12Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor.18c0b09c4418f480 namespace: kserve-ci-e2e-test resourceVersion: "17005" uid: b2473757-69d1-4f11-ac48-7ca9f95c5789 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T17:54:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-079fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16891" uid: 13f2c16f-d808-4991-bf33-6804010560fa kind: Event lastTimestamp: "2026-07-09T17:55:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:55:12Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor.18c0b0a34218b84f namespace: kserve-ci-e2e-test resourceVersion: "17468" uid: 997f9434-5a30-419c-8600-ccc811328092 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T17:54:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-079fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16891" uid: 13f2c16f-d808-4991-bf33-6804010560fa kind: Event lastTimestamp: "2026-07-09T17:55:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:55:12Z" name: isvc-sklearn-graph-raw-hpa-079fb-predictor.18c0b0a3428ca6c1 namespace: kserve-ci-e2e-test resourceVersion: "17469" uid: a3123f9e-2f89-4441-bb46-0fd2cf0608a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-09T17:53:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-079fb namespace: kserve-ci-e2e-test resourceVersion: "16627" uid: 15b79527-7d6b-412d-b122-2d504b51fb2c kind: Event lastTimestamp: "2026-07-09T17:54:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T17:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:54:58Z" name: isvc-sklearn-graph-raw-hpa-079fb.18c0b098c5335f4e namespace: kserve-ci-e2e-test resourceVersion: "17368" uid: 079484d5-c5f0-4770-9ca3-52fed2320d02 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-079fb namespace: kserve-ci-e2e-test resourceVersion: "16627" uid: 15b79527-7d6b-412d-b122-2d504b51fb2c kind: Event lastTimestamp: "2026-07-09T17:53:42Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-079fb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-079fb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T17:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:53:42Z" name: isvc-sklearn-graph-raw-hpa-079fb.18c0b098c591b557 namespace: kserve-ci-e2e-test resourceVersion: "16653" uid: 7a67a466-8b7c-4dd2-964a-51e2ad2e0f01 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:54:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-079fb namespace: kserve-ci-e2e-test resourceVersion: "17366" uid: 15b79527-7d6b-412d-b122-2d504b51fb2c kind: Event lastTimestamp: "2026-07-09T17:54:58Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-079fb] is Ready metadata: creationTimestamp: "2026-07-09T17:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:54:58Z" name: isvc-sklearn-graph-raw-hpa-079fb.18c0b0aa7623dba0 namespace: kserve-ci-e2e-test resourceVersion: "17367" uid: 5121d830-33ed-49bd-a615-f3bf67ffed74 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:57:18.429418Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9 namespace: kserve-ci-e2e-test resourceVersion: "18647" uid: 0eacc765-85e4-4319-88f6-88b545d56ac2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9 to ip-10-0-143-142.ec2.internal metadata: creationTimestamp: "2026-07-09T17:57:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:57:18Z" name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9.18c0b0cb03ed2b89 namespace: kserve-ci-e2e-test resourceVersion: "18652" uid: f5e4f94d-9901-43da-8bdc-a30de70f8d1f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-74fd97cb5b-77b77 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:18Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9 namespace: kserve-ci-e2e-test resourceVersion: "18649" uid: 0eacc765-85e4-4319-88f6-88b545d56ac2 kind: Event lastTimestamp: "2026-07-09T17:57:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-3cac6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T17:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:57:18Z" name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9.18c0b0cb123eafb9 namespace: kserve-ci-e2e-test resourceVersion: "18683" uid: 213581b5-fc33-4165-9270-78f7f7546173 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:19Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9 namespace: kserve-ci-e2e-test resourceVersion: "18668" uid: 0eacc765-85e4-4319-88f6-88b545d56ac2 kind: Event lastTimestamp: "2026-07-09T17:57:19Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:57:19Z" name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9.18c0b0cb40fada08 namespace: kserve-ci-e2e-test resourceVersion: "18695" uid: a048ee35-82e3-4dc8-8b68-28d11970cc25 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9 namespace: kserve-ci-e2e-test resourceVersion: "18649" uid: 0eacc765-85e4-4319-88f6-88b545d56ac2 kind: Event lastTimestamp: "2026-07-09T17:57:19Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6455858d9bcb13c4793d39d844d39ea45d90725b82962b5a6791b22458b6ca42" already present on machine metadata: creationTimestamp: "2026-07-09T17:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:57:19Z" name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9.18c0b0cb4299f008 namespace: kserve-ci-e2e-test resourceVersion: "18697" uid: 94b89e87-abf7-4e82-9a84-cd46af1e787d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9 namespace: kserve-ci-e2e-test resourceVersion: "18649" uid: 0eacc765-85e4-4319-88f6-88b545d56ac2 kind: Event lastTimestamp: "2026-07-09T17:57:19Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T17:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:57:19Z" name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9.18c0b0cb45aad15b namespace: kserve-ci-e2e-test resourceVersion: "18700" uid: 7cd371d0-b535-4dcd-acce-1de70d068f06 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9 namespace: kserve-ci-e2e-test resourceVersion: "18649" uid: 0eacc765-85e4-4319-88f6-88b545d56ac2 kind: Event lastTimestamp: "2026-07-09T17:57:19Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T17:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:57:19Z" name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9.18c0b0cb461ec3d0 namespace: kserve-ci-e2e-test resourceVersion: "18701" uid: 188f3b66-ed28-4a77-a4c8-d008fd0d1468 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9 namespace: kserve-ci-e2e-test resourceVersion: "18649" uid: 0eacc765-85e4-4319-88f6-88b545d56ac2 kind: Event lastTimestamp: "2026-07-09T17:57:24Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T17:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:57:24Z" name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9.18c0b0cc500125a1 namespace: kserve-ci-e2e-test resourceVersion: "18734" uid: e6611fb2-d517-4d2c-ba30-f2c7a8510300 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9 namespace: kserve-ci-e2e-test resourceVersion: "18649" uid: 0eacc765-85e4-4319-88f6-88b545d56ac2 kind: Event lastTimestamp: "2026-07-09T17:57:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T17:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:57:24Z" name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9.18c0b0cc52f44e0e namespace: kserve-ci-e2e-test resourceVersion: "18736" uid: 2658c8a0-574d-4972-b753-1177fc1e67ec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9 namespace: kserve-ci-e2e-test resourceVersion: "18649" uid: 0eacc765-85e4-4319-88f6-88b545d56ac2 kind: Event lastTimestamp: "2026-07-09T17:57:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T17:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:57:24Z" name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9.18c0b0cc536d572c namespace: kserve-ci-e2e-test resourceVersion: "18737" uid: 23f48b24-1bba-4eae-805c-e7224e349fb2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9 namespace: kserve-ci-e2e-test resourceVersion: "18649" uid: 0eacc765-85e4-4319-88f6-88b545d56ac2 kind: Event lastTimestamp: "2026-07-09T17:57:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T17:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:57:24Z" name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9.18c0b0cc5378bdb7 namespace: kserve-ci-e2e-test resourceVersion: "18740" uid: 321e8a52-a73a-4238-8a2d-2fb8c26ef6e3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9 namespace: kserve-ci-e2e-test resourceVersion: "18649" uid: 0eacc765-85e4-4319-88f6-88b545d56ac2 kind: Event lastTimestamp: "2026-07-09T17:57:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T17:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:57:24Z" name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9.18c0b0cc565e406e namespace: kserve-ci-e2e-test resourceVersion: "18742" uid: 54da341c-454f-4e3d-9c3e-688b298a2f05 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9 namespace: kserve-ci-e2e-test resourceVersion: "18649" uid: 0eacc765-85e4-4319-88f6-88b545d56ac2 kind: Event lastTimestamp: "2026-07-09T17:57:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:57:24Z" name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9.18c0b0cc56d30768 namespace: kserve-ci-e2e-test resourceVersion: "18743" uid: c2ef6717-6cf2-4417-ad17-47b7815ffd5b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 18 eventTime: null firstTimestamp: "2026-07-09T17:57:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9 namespace: kserve-ci-e2e-test resourceVersion: "18649" uid: 0eacc765-85e4-4319-88f6-88b545d56ac2 kind: Event lastTimestamp: "2026-07-09T18:00:01Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:57:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:01Z" name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9.18c0b0cc8c1448ab namespace: kserve-ci-e2e-test resourceVersion: "19888" uid: da058b45-d618-4fed-a636-8dc4b0138cea reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9 namespace: kserve-ci-e2e-test resourceVersion: "18649" uid: 0eacc765-85e4-4319-88f6-88b545d56ac2 kind: Event lastTimestamp: "2026-07-09T17:59:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T17:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:58Z" name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9.18c0b0f04dadb94f namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: 6a304dfa-e614-4643-b0dc-79b6b827a377 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9 namespace: kserve-ci-e2e-test resourceVersion: "18649" uid: 0eacc765-85e4-4319-88f6-88b545d56ac2 kind: Event lastTimestamp: "2026-07-09T17:59:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:59:58Z" name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9.18c0b0f04db0101a namespace: kserve-ci-e2e-test resourceVersion: "19863" uid: ba8e207b-64f7-4eca-95e3-f6a2a6e4a6e1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:00:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9 namespace: kserve-ci-e2e-test resourceVersion: "18649" uid: 0eacc765-85e4-4319-88f6-88b545d56ac2 kind: Event lastTimestamp: "2026-07-09T18:00:06Z" message: 'Readiness probe failed: Get "https://10.132.0.30:8643/healthz": dial tcp 10.132.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:00:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:00:06Z" name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9.18c0b0f0de9ed989 namespace: kserve-ci-e2e-test resourceVersion: "19967" uid: 0879ae45-3a40-4443-b425-f35f96955d56 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc namespace: kserve-ci-e2e-test resourceVersion: "18629" uid: 58a46d7d-a942-41cb-9517-4608bd559695 kind: Event lastTimestamp: "2026-07-09T17:57:18Z" message: 'Created pod: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc-lv8p9' metadata: creationTimestamp: "2026-07-09T17:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:57:18Z" name: isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc.18c0b0cb037f6221 namespace: kserve-ci-e2e-test resourceVersion: "18648" uid: 0dec5ba0-a08b-42d5-9fd6-faae4e6e0d47 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-3cac6-predictor namespace: kserve-ci-e2e-test resourceVersion: "18622" uid: a1a2a3d8-fbd5-4ba1-88a0-1c5cfab8643b kind: Event lastTimestamp: "2026-07-09T17:57:18Z" message: Scaled up replica set isvc-sklearn-scale-raw-3cac6-predictor-85cd6d56dc from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:57:18Z" name: isvc-sklearn-scale-raw-3cac6-predictor.18c0b0cafee8ac92 namespace: kserve-ci-e2e-test resourceVersion: "18630" uid: 96275bdc-d5e9-40f2-b16e-8fdd0c2a68d1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:57:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-3cac6-predictor namespace: kserve-ci-e2e-test resourceVersion: "18628" uid: cda0a3fa-fcdc-4e96-ade2-7b34357c39ca kind: Event lastTimestamp: "2026-07-09T17:58:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:58:03Z" name: isvc-sklearn-scale-raw-3cac6-predictor.18c0b0ce7db9eea7 namespace: kserve-ci-e2e-test resourceVersion: "19041" uid: da6e731d-cf43-4094-b9d0-38ae22bebbc3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:57:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-3cac6-predictor namespace: kserve-ci-e2e-test resourceVersion: "18628" uid: cda0a3fa-fcdc-4e96-ade2-7b34357c39ca kind: Event lastTimestamp: "2026-07-09T17:58:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:57:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:58:03Z" name: isvc-sklearn-scale-raw-3cac6-predictor.18c0b0ce7e230e1f namespace: kserve-ci-e2e-test resourceVersion: "19042" uid: 61cd66c2-ceee-4c6f-9811-e6a4a48fe52b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-09T17:57:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-3cac6-predictor namespace: kserve-ci-e2e-test resourceVersion: "18822" uid: cda0a3fa-fcdc-4e96-ade2-7b34357c39ca kind: Event lastTimestamp: "2026-07-09T17:59:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:59:48Z" name: isvc-sklearn-scale-raw-3cac6-predictor.18c0b0d1fcdf764e namespace: kserve-ci-e2e-test resourceVersion: "19768" uid: 6aea3cc4-af52-49b4-9144-ba8d59797944 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-09T17:57:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-3cac6-predictor namespace: kserve-ci-e2e-test resourceVersion: "18822" uid: cda0a3fa-fcdc-4e96-ade2-7b34357c39ca kind: Event lastTimestamp: "2026-07-09T17:59:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:59:48Z" name: isvc-sklearn-scale-raw-3cac6-predictor.18c0b0d1fd49a2ee namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 94a0c730-d47e-4cfc-a5de-f8322fca555e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-09T17:57:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-3cac6 namespace: kserve-ci-e2e-test resourceVersion: "18605" uid: 8233bff4-b75c-49c9-aa46-d3b7a58f13a5 kind: Event lastTimestamp: "2026-07-09T17:59:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T17:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:59:51Z" name: isvc-sklearn-scale-raw-3cac6.18c0b0cafee94ec5 namespace: kserve-ci-e2e-test resourceVersion: "19801" uid: c34105ba-8bc8-4e6d-a83a-94f24fc8085d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-3cac6 namespace: kserve-ci-e2e-test resourceVersion: "18605" uid: 8233bff4-b75c-49c9-aa46-d3b7a58f13a5 kind: Event lastTimestamp: "2026-07-09T17:57:18Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-3cac6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-3cac6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T17:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:57:18Z" name: isvc-sklearn-scale-raw-3cac6.18c0b0caff627c75 namespace: kserve-ci-e2e-test resourceVersion: "18634" uid: 67d6248f-f6ef-4999-93f8-acd1e61dff59 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:59:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-3cac6 namespace: kserve-ci-e2e-test resourceVersion: "19799" uid: 8233bff4-b75c-49c9-aa46-d3b7a58f13a5 kind: Event lastTimestamp: "2026-07-09T17:59:51Z" message: InferenceService [isvc-sklearn-scale-raw-3cac6] is Ready metadata: creationTimestamp: "2026-07-09T17:59:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:59:51Z" name: isvc-sklearn-scale-raw-3cac6.18c0b0eeaed31fa8 namespace: kserve-ci-e2e-test resourceVersion: "19800" uid: ef453a3c-9e40-4538-ba2b-2dde54b7be93 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:51:52.473496Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m namespace: kserve-ci-e2e-test resourceVersion: "15671" uid: 7288eb6d-3a1c-48b3-97c0-3a62a47795a9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m to ip-10-0-143-142.ec2.internal metadata: creationTimestamp: "2026-07-09T17:51:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:51:52Z" name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m.18c0b07f1f7085d6 namespace: kserve-ci-e2e-test resourceVersion: "15677" uid: 801a08d1-dd98-468c-9dea-2a1b2dedfb86 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-74fd97cb5b-77b77 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: 7288eb6d-3a1c-48b3-97c0-3a62a47795a9 kind: Event lastTimestamp: "2026-07-09T17:51:52Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:51:52Z" name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m.18c0b07f397626f2 namespace: kserve-ci-e2e-test resourceVersion: "15698" uid: 788e5db6-76a4-41aa-881d-47ba0eb0a820 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 7288eb6d-3a1c-48b3-97c0-3a62a47795a9 kind: Event lastTimestamp: "2026-07-09T17:51:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6455858d9bcb13c4793d39d844d39ea45d90725b82962b5a6791b22458b6ca42" already present on machine metadata: creationTimestamp: "2026-07-09T17:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:52Z" name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m.18c0b07f3afb0290 namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: 4bec3d5b-4aa4-4799-bb9c-45ccbd26f10d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 7288eb6d-3a1c-48b3-97c0-3a62a47795a9 kind: Event lastTimestamp: "2026-07-09T17:51:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T17:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:52Z" name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m.18c0b07f3e3f8777 namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: b5e4dca2-362a-4a28-a68f-f54af80a1e51 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 7288eb6d-3a1c-48b3-97c0-3a62a47795a9 kind: Event lastTimestamp: "2026-07-09T17:51:52Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T17:51:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:53Z" name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m.18c0b07f3eb678f7 namespace: kserve-ci-e2e-test resourceVersion: "15703" uid: db92faa3-a12a-4781-8034-e54a125d407a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 7288eb6d-3a1c-48b3-97c0-3a62a47795a9 kind: Event lastTimestamp: "2026-07-09T17:51:57Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-09T17:51:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:51:57Z" name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m.18c0b08052bd3003 namespace: kserve-ci-e2e-test resourceVersion: "15767" uid: af7554d1-d4e2-4a26-8dbf-38a9269feab8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:52:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 7288eb6d-3a1c-48b3-97c0-3a62a47795a9 kind: Event lastTimestamp: "2026-07-09T17:52:16Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 18.712s (18.712s including waiting). Image size: 1305579259 bytes.' metadata: creationTimestamp: "2026-07-09T17:52:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:52:16Z" name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m.18c0b084ae12d8ba namespace: kserve-ci-e2e-test resourceVersion: "15914" uid: cdacb298-eb76-4a86-94b4-f9bdba211fae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:52:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 7288eb6d-3a1c-48b3-97c0-3a62a47795a9 kind: Event lastTimestamp: "2026-07-09T17:52:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T17:52:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:52:16Z" name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m.18c0b084b1a087bd namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: 3268afe1-b857-43bb-8244-7d39122e116e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:52:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 7288eb6d-3a1c-48b3-97c0-3a62a47795a9 kind: Event lastTimestamp: "2026-07-09T17:52:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T17:52:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:52:16Z" name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m.18c0b084b2107aaf namespace: kserve-ci-e2e-test resourceVersion: "15916" uid: 1d01254c-df45-4c88-a1f2-7968b6035a18 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:52:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 7288eb6d-3a1c-48b3-97c0-3a62a47795a9 kind: Event lastTimestamp: "2026-07-09T17:52:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T17:52:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:52:16Z" name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m.18c0b084b21a2e4e namespace: kserve-ci-e2e-test resourceVersion: "15917" uid: a4b92b01-b393-44a5-8c25-f57f5521131d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:52:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 7288eb6d-3a1c-48b3-97c0-3a62a47795a9 kind: Event lastTimestamp: "2026-07-09T17:52:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T17:52:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:52:16Z" name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m.18c0b084b5580fac namespace: kserve-ci-e2e-test resourceVersion: "15918" uid: 19bb0daa-edaf-4cc9-ae04-5cc8e6d27107 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:52:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 7288eb6d-3a1c-48b3-97c0-3a62a47795a9 kind: Event lastTimestamp: "2026-07-09T17:52:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:52:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:52:16Z" name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m.18c0b084b5d20b05 namespace: kserve-ci-e2e-test resourceVersion: "15919" uid: b53369d2-f861-4078-980f-f2f1b00a5a27 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-09T17:52:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 7288eb6d-3a1c-48b3-97c0-3a62a47795a9 kind: Event lastTimestamp: "2026-07-09T17:53:43Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:52:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:43Z" name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m.18c0b0850063cd22 namespace: kserve-ci-e2e-test resourceVersion: "16746" uid: 332d2db4-742a-4d39-a82f-aceadcdf254b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 7288eb6d-3a1c-48b3-97c0-3a62a47795a9 kind: Event lastTimestamp: "2026-07-09T17:53:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T17:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:42Z" name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m.18c0b098cde9527d namespace: kserve-ci-e2e-test resourceVersion: "16701" uid: 2726535a-aed2-41aa-a617-ccdcbf975e22 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 7288eb6d-3a1c-48b3-97c0-3a62a47795a9 kind: Event lastTimestamp: "2026-07-09T17:53:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:42Z" name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m.18c0b098cde9c619 namespace: kserve-ci-e2e-test resourceVersion: "16704" uid: 2645a11d-88dd-4581-b759-42a87aad6563 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 7288eb6d-3a1c-48b3-97c0-3a62a47795a9 kind: Event lastTimestamp: "2026-07-09T17:53:43Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:43Z" name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m.18c0b0990692f3dc namespace: kserve-ci-e2e-test resourceVersion: "16745" uid: cfa5713c-0715-4fa3-8d4f-2e0245a8ba2a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf namespace: kserve-ci-e2e-test resourceVersion: "15658" uid: bfaa9d2c-bc9b-475d-97d3-a20e3ccedef7 kind: Event lastTimestamp: "2026-07-09T17:51:52Z" message: 'Created pod: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf-58k8m' metadata: creationTimestamp: "2026-07-09T17:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:52Z" name: isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf.18c0b07f1ef35d01 namespace: kserve-ci-e2e-test resourceVersion: "15673" uid: b6fbede5-3089-49cf-9a36-c123f8dde23c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:51:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-02e34-predictor namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: a6206d4c-11d7-4a7f-afdf-42668cf69d43 kind: Event lastTimestamp: "2026-07-09T17:51:52Z" message: Scaled up replica set isvc-xgboost-graph-raw-02e34-predictor-59c4699bdf from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:51:52Z" name: isvc-xgboost-graph-raw-02e34-predictor.18c0b07f1b7f401f namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: d2a5fca7-6453-4a0c-b9bb-323129cd3ba1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T17:52:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-02e34-predictor namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: d0e51d50-6980-4c12-bfd1-acd2017f807b kind: Event lastTimestamp: "2026-07-09T17:52:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:52:52Z" name: isvc-xgboost-graph-raw-02e34-predictor.18c0b0829a97155f namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: 1a9e7332-da18-4eb8-bc9f-5015a51fb481 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T17:52:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-02e34-predictor namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: d0e51d50-6980-4c12-bfd1-acd2017f807b kind: Event lastTimestamp: "2026-07-09T17:52:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:52:52Z" name: isvc-xgboost-graph-raw-02e34-predictor.18c0b0829b47bfa5 namespace: kserve-ci-e2e-test resourceVersion: "16188" uid: d320c18b-990f-4398-8fc5-fa1635abfe5a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:53:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-02e34-predictor namespace: kserve-ci-e2e-test resourceVersion: "15846" uid: d0e51d50-6980-4c12-bfd1-acd2017f807b kind: Event lastTimestamp: "2026-07-09T17:53:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:53:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:53:37Z" name: isvc-xgboost-graph-raw-02e34-predictor.18c0b090968b726f namespace: kserve-ci-e2e-test resourceVersion: "16566" uid: 7c615f70-bee5-4102-975c-642cb0cc2723 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-09T17:53:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-02e34-predictor namespace: kserve-ci-e2e-test resourceVersion: "15846" uid: d0e51d50-6980-4c12-bfd1-acd2017f807b kind: Event lastTimestamp: "2026-07-09T17:53:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:53:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:53:37Z" name: isvc-xgboost-graph-raw-02e34-predictor.18c0b09096eb720d namespace: kserve-ci-e2e-test resourceVersion: "16567" uid: fdc247e3-dde1-44d5-b5ae-32a7df187368 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-09T17:51:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-02e34 namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: b33331fc-3ee0-4a29-9849-5ac0874ccdfe kind: Event lastTimestamp: "2026-07-09T17:53:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T17:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:53:23Z" name: isvc-xgboost-graph-raw-02e34.18c0b07f1bc1ef67 namespace: kserve-ci-e2e-test resourceVersion: "16435" uid: 777f56f3-24d2-48be-b402-1f380710023c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-02e34 namespace: kserve-ci-e2e-test resourceVersion: "16433" uid: b33331fc-3ee0-4a29-9849-5ac0874ccdfe kind: Event lastTimestamp: "2026-07-09T17:53:23Z" message: InferenceService [isvc-xgboost-graph-raw-02e34] is Ready metadata: creationTimestamp: "2026-07-09T17:53:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:53:23Z" name: isvc-xgboost-graph-raw-02e34.18c0b09464694edf namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: 409d6098-0c11-48e9-94f5-1585df541ecf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:53:42.797347Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q namespace: kserve-ci-e2e-test resourceVersion: "16703" uid: 0e7a722b-3775-4c66-ab4c-af3000ef39ef kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q to ip-10-0-143-142.ec2.internal metadata: creationTimestamp: "2026-07-09T17:53:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:53:42Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q.18c0b098cf40aba9 namespace: kserve-ci-e2e-test resourceVersion: "16710" uid: eb1f8546-6cd1-497d-aa5e-4adbf404aa21 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-74fd97cb5b-77b77 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 0e7a722b-3775-4c66-ab4c-af3000ef39ef kind: Event lastTimestamp: "2026-07-09T17:53:43Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-raw-hpa-079fb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T17:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:43Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q.18c0b098de9ae61d namespace: kserve-ci-e2e-test resourceVersion: "16729" uid: f19c3659-5db5-4de5-99bf-28493eec5d3f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q namespace: kserve-ci-e2e-test resourceVersion: "16718" uid: 0e7a722b-3775-4c66-ab4c-af3000ef39ef kind: Event lastTimestamp: "2026-07-09T17:53:43Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:53:43Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q.18c0b0990d21d1bd namespace: kserve-ci-e2e-test resourceVersion: "16750" uid: 2b70beb1-c869-4d8a-b8ee-b4703b425be6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 0e7a722b-3775-4c66-ab4c-af3000ef39ef kind: Event lastTimestamp: "2026-07-09T17:53:43Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6455858d9bcb13c4793d39d844d39ea45d90725b82962b5a6791b22458b6ca42" already present on machine metadata: creationTimestamp: "2026-07-09T17:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:43Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q.18c0b0990ffbaf93 namespace: kserve-ci-e2e-test resourceVersion: "16752" uid: 4af75235-6e50-4a1c-8233-333938899ada reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 0e7a722b-3775-4c66-ab4c-af3000ef39ef kind: Event lastTimestamp: "2026-07-09T17:53:43Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T17:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:43Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q.18c0b099132461a9 namespace: kserve-ci-e2e-test resourceVersion: "16754" uid: 926a9762-11a6-415d-b06f-b2c4191e4bbd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 0e7a722b-3775-4c66-ab4c-af3000ef39ef kind: Event lastTimestamp: "2026-07-09T17:53:43Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T17:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:43Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q.18c0b0991393e6b2 namespace: kserve-ci-e2e-test resourceVersion: "16755" uid: 897a8533-bba8-444d-b911-a1032ad2863d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 0e7a722b-3775-4c66-ab4c-af3000ef39ef kind: Event lastTimestamp: "2026-07-09T17:53:48Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-07-09T17:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:48Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q.18c0b09a0b6de310 namespace: kserve-ci-e2e-test resourceVersion: "16807" uid: 8edfa706-0d02-440c-a5ce-7b6109ac1161 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 0e7a722b-3775-4c66-ab4c-af3000ef39ef kind: Event lastTimestamp: "2026-07-09T17:53:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T17:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:48Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q.18c0b09a0f40d785 namespace: kserve-ci-e2e-test resourceVersion: "16812" uid: 60dc8926-ba57-473e-ac57-c70d0ce0c309 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 0e7a722b-3775-4c66-ab4c-af3000ef39ef kind: Event lastTimestamp: "2026-07-09T17:53:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T17:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:48Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q.18c0b09a0fb4090a namespace: kserve-ci-e2e-test resourceVersion: "16813" uid: fcbbc52a-80d5-4bdd-9773-20f934a4546d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 0e7a722b-3775-4c66-ab4c-af3000ef39ef kind: Event lastTimestamp: "2026-07-09T17:53:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T17:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:48Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q.18c0b09a0fbf1c72 namespace: kserve-ci-e2e-test resourceVersion: "16814" uid: 293d71a0-dc87-4d8c-8017-343830a2a6b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 0e7a722b-3775-4c66-ab4c-af3000ef39ef kind: Event lastTimestamp: "2026-07-09T17:53:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T17:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:48Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q.18c0b09a125059b2 namespace: kserve-ci-e2e-test resourceVersion: "16817" uid: b296062d-8e8e-47ff-bc04-ea9bb348e77d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 0e7a722b-3775-4c66-ab4c-af3000ef39ef kind: Event lastTimestamp: "2026-07-09T17:53:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:48Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q.18c0b09a12c426a0 namespace: kserve-ci-e2e-test resourceVersion: "16818" uid: 8845f179-a87e-4801-bcc2-35429ff4b35f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-09T17:53:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 0e7a722b-3775-4c66-ab4c-af3000ef39ef kind: Event lastTimestamp: "2026-07-09T17:55:25Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:53:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:25Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q.18c0b09a47ad463d namespace: kserve-ci-e2e-test resourceVersion: "17693" uid: 18ea58a0-8754-43db-a85a-29dbf3176ceb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 0e7a722b-3775-4c66-ab4c-af3000ef39ef kind: Event lastTimestamp: "2026-07-09T17:55:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T17:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:23Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q.18c0b0b02cfde1fa namespace: kserve-ci-e2e-test resourceVersion: "17663" uid: 4675e6e0-7368-4d57-abbb-eef1cdd7f758 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 0e7a722b-3775-4c66-ab4c-af3000ef39ef kind: Event lastTimestamp: "2026-07-09T17:55:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:23Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q.18c0b0b02cfe4d3b namespace: kserve-ci-e2e-test resourceVersion: "17664" uid: 6a9d7ffd-7aea-4968-a557-5e9562880361 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 0e7a722b-3775-4c66-ab4c-af3000ef39ef kind: Event lastTimestamp: "2026-07-09T17:55:25Z" message: 'Readiness probe failed: Get "https://10.132.0.27:8643/healthz": dial tcp 10.132.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:25Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q.18c0b0b0a206ade2 namespace: kserve-ci-e2e-test resourceVersion: "17691" uid: 0fee566f-47d2-410f-8f8d-34ebd6994873 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9 namespace: kserve-ci-e2e-test resourceVersion: "16677" uid: 95ad54e7-fdcd-4b57-b245-423791d943a3 kind: Event lastTimestamp: "2026-07-09T17:53:42Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9-kfj7q' metadata: creationTimestamp: "2026-07-09T17:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:53:42Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9.18c0b098ceb549dc namespace: kserve-ci-e2e-test resourceVersion: "16705" uid: 19336c6e-be46-4cdf-b83c-164b5d5ef86b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-079fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16671" uid: c7730f64-f363-4e8c-a0ab-71acd15938ef kind: Event lastTimestamp: "2026-07-09T17:53:42Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-079fb-predictor-5975dc47d9 from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:53:42Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor.18c0b098cae6370f namespace: kserve-ci-e2e-test resourceVersion: "16679" uid: 58f52a7f-fd3e-467d-8afd-5c999434f2fc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:53:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-079fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16675" uid: d05c2927-fef3-4378-98c1-92dfe771b684 kind: Event lastTimestamp: "2026-07-09T17:54:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:54:12Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor.18c0b09c4971e924 namespace: kserve-ci-e2e-test resourceVersion: "17007" uid: 1e704f31-f215-4db1-b64d-fd3032733c31 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:53:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-079fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16675" uid: d05c2927-fef3-4378-98c1-92dfe771b684 kind: Event lastTimestamp: "2026-07-09T17:54:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:54:12Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor.18c0b09c49dfc66e namespace: kserve-ci-e2e-test resourceVersion: "17008" uid: b1a037fa-3ea4-43ff-9f5b-4cd8a4cf65b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T17:54:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-079fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16895" uid: d05c2927-fef3-4378-98c1-92dfe771b684 kind: Event lastTimestamp: "2026-07-09T17:55:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:55:12Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor.18c0b0a34731c2ed namespace: kserve-ci-e2e-test resourceVersion: "17481" uid: f4889b82-6099-446d-9d53-8fa5aca2547f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T17:54:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-079fb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16895" uid: d05c2927-fef3-4378-98c1-92dfe771b684 kind: Event lastTimestamp: "2026-07-09T17:55:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T17:54:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:55:12Z" name: isvc-xgboost-graph-raw-hpa-079fb-predictor.18c0b0a347b3e650 namespace: kserve-ci-e2e-test resourceVersion: "17482" uid: 9357ecef-4504-44a2-9b74-e0e60e210fc4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-09T17:53:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-079fb namespace: kserve-ci-e2e-test resourceVersion: "16660" uid: 3b8ba32b-6f26-45a1-b69a-6cc7527f7018 kind: Event lastTimestamp: "2026-07-09T17:54:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T17:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:54:55Z" name: isvc-xgboost-graph-raw-hpa-079fb.18c0b098cb1a9baa namespace: kserve-ci-e2e-test resourceVersion: "17325" uid: 17cc3fb4-2cdf-4653-bf95-8a7ecfeba8c7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:54:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-079fb namespace: kserve-ci-e2e-test resourceVersion: "17323" uid: 3b8ba32b-6f26-45a1-b69a-6cc7527f7018 kind: Event lastTimestamp: "2026-07-09T17:54:55Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-079fb] is Ready metadata: creationTimestamp: "2026-07-09T17:54:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:54:55Z" name: isvc-xgboost-graph-raw-hpa-079fb.18c0b0a9ab2b275b namespace: kserve-ci-e2e-test resourceVersion: "17324" uid: 0f1e7351-e1e2-45af-a602-1b638153358a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:55:23.067013Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r namespace: kserve-ci-e2e-test resourceVersion: "17630" uid: 34b2e932-3477-4cbe-868b-f967dd800fc9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r to ip-10-0-143-142.ec2.internal metadata: creationTimestamp: "2026-07-09T17:55:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:55:23Z" name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r.18c0b0b027ca602a namespace: kserve-ci-e2e-test resourceVersion: "17636" uid: 048531d9-a0de-4925-bc02-fd2fa141bf6c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-74fd97cb5b-77b77 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:23Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r namespace: kserve-ci-e2e-test resourceVersion: "17642" uid: 34b2e932-3477-4cbe-868b-f967dd800fc9 kind: Event lastTimestamp: "2026-07-09T17:55:23Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:55:23Z" name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r.18c0b0b0410c37cb namespace: kserve-ci-e2e-test resourceVersion: "17669" uid: ea2322e6-5879-43eb-ad39-20565d7f3949 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r namespace: kserve-ci-e2e-test resourceVersion: "17634" uid: 34b2e932-3477-4cbe-868b-f967dd800fc9 kind: Event lastTimestamp: "2026-07-09T17:55:23Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-09T17:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:23Z" name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r.18c0b0b04265b8b1 namespace: kserve-ci-e2e-test resourceVersion: "17672" uid: 50bf987c-f75f-49d9-873f-9fa261b10242 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r namespace: kserve-ci-e2e-test resourceVersion: "17634" uid: 34b2e932-3477-4cbe-868b-f967dd800fc9 kind: Event lastTimestamp: "2026-07-09T17:55:24Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.357s (1.358s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-09T17:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:24Z" name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r.18c0b0b09357ba12 namespace: kserve-ci-e2e-test resourceVersion: "17684" uid: cda67414-b6e4-4fd8-aaf9-d687fbaf73d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r namespace: kserve-ci-e2e-test resourceVersion: "17634" uid: 34b2e932-3477-4cbe-868b-f967dd800fc9 kind: Event lastTimestamp: "2026-07-09T17:55:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T17:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:24Z" name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r.18c0b0b096d3557b namespace: kserve-ci-e2e-test resourceVersion: "17685" uid: fcf0d3a4-2b5b-40f6-ab81-79c33e64d909 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r namespace: kserve-ci-e2e-test resourceVersion: "17634" uid: 34b2e932-3477-4cbe-868b-f967dd800fc9 kind: Event lastTimestamp: "2026-07-09T17:55:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T17:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:24Z" name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r.18c0b0b09741b4bb namespace: kserve-ci-e2e-test resourceVersion: "17686" uid: 2ceadc6c-0e3a-4688-8e7a-fcc78b4bff6d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r namespace: kserve-ci-e2e-test resourceVersion: "17634" uid: 34b2e932-3477-4cbe-868b-f967dd800fc9 kind: Event lastTimestamp: "2026-07-09T17:55:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T17:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:24Z" name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r.18c0b0b0974ba80d namespace: kserve-ci-e2e-test resourceVersion: "17687" uid: 6a931467-4675-45f4-8d60-149f540b1b1c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r namespace: kserve-ci-e2e-test resourceVersion: "17634" uid: 34b2e932-3477-4cbe-868b-f967dd800fc9 kind: Event lastTimestamp: "2026-07-09T17:55:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T17:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:24Z" name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r.18c0b0b09a520ec7 namespace: kserve-ci-e2e-test resourceVersion: "17688" uid: 84dbd3c4-1054-4ebd-8a3f-b1454111ae5f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r namespace: kserve-ci-e2e-test resourceVersion: "17634" uid: 34b2e932-3477-4cbe-868b-f967dd800fc9 kind: Event lastTimestamp: "2026-07-09T17:55:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:55:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:24Z" name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r.18c0b0b09abf4c2b namespace: kserve-ci-e2e-test resourceVersion: "17689" uid: e74da3c1-d988-42f6-b5ea-9b727a31846d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r namespace: kserve-ci-e2e-test resourceVersion: "17634" uid: 34b2e932-3477-4cbe-868b-f967dd800fc9 kind: Event lastTimestamp: "2026-07-09T17:57:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T17:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:57:18Z" name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r.18c0b0cb06ab14da namespace: kserve-ci-e2e-test resourceVersion: "18672" uid: 27420460-5f7f-474a-ae15-ee451f58a0df reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r namespace: kserve-ci-e2e-test resourceVersion: "17634" uid: 34b2e932-3477-4cbe-868b-f967dd800fc9 kind: Event lastTimestamp: "2026-07-09T17:57:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T17:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:57:18Z" name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r.18c0b0cb06ab7371 namespace: kserve-ci-e2e-test resourceVersion: "18673" uid: 655bb7f0-5030-40f5-9696-6307d9594cdb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:57:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r namespace: kserve-ci-e2e-test resourceVersion: "17634" uid: 34b2e932-3477-4cbe-868b-f967dd800fc9 kind: Event lastTimestamp: "2026-07-09T17:57:18Z" message: 'Readiness probe failed: Get "https://10.132.0.28:8643/healthz": dial tcp 10.132.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:57:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:57:18Z" name: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r.18c0b0cb0a0dcd94 namespace: kserve-ci-e2e-test resourceVersion: "18680" uid: b0b9697f-cfde-49ed-9897-de5b56038bf6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-1ffb5-predictor-69db84bc5d namespace: kserve-ci-e2e-test resourceVersion: "17613" uid: 33355828-2520-47dd-adb8-7af3b7068b67 kind: Event lastTimestamp: "2026-07-09T17:55:23Z" message: 'Created pod: message-dumper-raw-1ffb5-predictor-69db84bc5d-k2k5r' metadata: creationTimestamp: "2026-07-09T17:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:55:23Z" name: message-dumper-raw-1ffb5-predictor-69db84bc5d.18c0b0b0275d7c91 namespace: kserve-ci-e2e-test resourceVersion: "17633" uid: c9894445-803f-45b7-bad1-bd052499be60 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-1ffb5-predictor namespace: kserve-ci-e2e-test resourceVersion: "17606" uid: 803d7afa-efcb-4a09-ade3-a140dbcfeda8 kind: Event lastTimestamp: "2026-07-09T17:55:23Z" message: Scaled up replica set message-dumper-raw-1ffb5-predictor-69db84bc5d from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:55:23Z" name: message-dumper-raw-1ffb5-predictor.18c0b0b024036aca namespace: kserve-ci-e2e-test resourceVersion: "17616" uid: c317fc0f-fa76-47be-b866-785f44a53696 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T17:55:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-1ffb5-predictor namespace: kserve-ci-e2e-test resourceVersion: "17610" uid: b751b37a-4089-46f7-aa90-b9697d66bc25 kind: Event lastTimestamp: "2026-07-09T17:56:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:55:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:56:23Z" name: message-dumper-raw-1ffb5-predictor.18c0b0b3a2e725fa namespace: kserve-ci-e2e-test resourceVersion: "18200" uid: 7cd2234a-88c8-4ced-861b-e77089a2c977 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T17:55:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-1ffb5-predictor namespace: kserve-ci-e2e-test resourceVersion: "17610" uid: b751b37a-4089-46f7-aa90-b9697d66bc25 kind: Event lastTimestamp: "2026-07-09T17:56:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T17:55:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:56:23Z" name: message-dumper-raw-1ffb5-predictor.18c0b0b3a343d5f4 namespace: kserve-ci-e2e-test resourceVersion: "18201" uid: f6eecbd3-8fae-4f29-9458-be7d1c15fa10 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-09T17:55:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-1ffb5 namespace: kserve-ci-e2e-test resourceVersion: "17595" uid: 93a19f30-e130-48dd-938a-a65409deb371 kind: Event lastTimestamp: "2026-07-09T17:55:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T17:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:55:33Z" name: message-dumper-raw-1ffb5.18c0b0b0242bba9b namespace: kserve-ci-e2e-test resourceVersion: "17765" uid: 9726a15d-ad54-4d49-bad8-537f895322d7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:55:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-1ffb5 namespace: kserve-ci-e2e-test resourceVersion: "17595" uid: 93a19f30-e130-48dd-938a-a65409deb371 kind: Event lastTimestamp: "2026-07-09T17:55:23Z" message: 'Failed to update status for InferenceService "message-dumper-raw-1ffb5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-1ffb5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T17:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:55:23Z" name: message-dumper-raw-1ffb5.18c0b0b024a94b53 namespace: kserve-ci-e2e-test resourceVersion: "17657" uid: 493d9661-2de4-4e17-9539-cec123b1f9af reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-1ffb5 namespace: kserve-ci-e2e-test resourceVersion: "17611" uid: 93a19f30-e130-48dd-938a-a65409deb371 kind: Event lastTimestamp: "2026-07-09T17:55:23Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-1ffb5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T17:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:55:23Z" name: message-dumper-raw-1ffb5.18c0b0b02a8c57ed namespace: kserve-ci-e2e-test resourceVersion: "17659" uid: aafc1191-b3c9-4154-8e4b-1c5a1dbb5d0c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-1ffb5 namespace: kserve-ci-e2e-test resourceVersion: "17763" uid: 93a19f30-e130-48dd-938a-a65409deb371 kind: Event lastTimestamp: "2026-07-09T17:55:33Z" message: InferenceService [message-dumper-raw-1ffb5] is Ready metadata: creationTimestamp: "2026-07-09T17:55:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:55:33Z" name: message-dumper-raw-1ffb5.18c0b0b29cb4954d namespace: kserve-ci-e2e-test resourceVersion: "17764" uid: fbfcdc8c-4df8-4859-b5ef-d92619ac67f5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:53:32.419316Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-02e34-699f4f88fd-9twms namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: a6551cde-8a10-43ca-8c12-cdb8730a9ad6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-02e34-699f4f88fd-9twms to ip-10-0-135-252.ec2.internal metadata: creationTimestamp: "2026-07-09T17:53:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:53:32Z" name: model-chainer-raw-02e34-699f4f88fd-9twms.18c0b09664ac755d namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: 890aca75-99c8-47be-87f3-0e626fdc155c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-74fd97cb5b-77b77 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:32Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-02e34-699f4f88fd-9twms namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: a6551cde-8a10-43ca-8c12-cdb8730a9ad6 kind: Event lastTimestamp: "2026-07-09T17:53:32Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-02e34-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T17:53:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:32Z" name: model-chainer-raw-02e34-699f4f88fd-9twms.18c0b0966f2a5173 namespace: kserve-ci-e2e-test resourceVersion: "16520" uid: 5d54d656-716e-4474-92c5-b7bdf05334a3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-252.ec2.internal source: component: kubelet host: ip-10-0-135-252.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:33Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-02e34-699f4f88fd-9twms namespace: kserve-ci-e2e-test resourceVersion: "16514" uid: a6551cde-8a10-43ca-8c12-cdb8730a9ad6 kind: Event lastTimestamp: "2026-07-09T17:53:33Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:53:33Z" name: model-chainer-raw-02e34-699f4f88fd-9twms.18c0b096a2417055 namespace: kserve-ci-e2e-test resourceVersion: "16528" uid: b615e712-c6ae-4eff-93cc-c7c107338604 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-02e34} kind: Pod name: model-chainer-raw-02e34-699f4f88fd-9twms namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: a6551cde-8a10-43ca-8c12-cdb8730a9ad6 kind: Event lastTimestamp: "2026-07-09T17:53:33Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:f94a36664a6553b816dc04c45e472a62cdcdb731a5f667f29ab4b59a2331a34e" metadata: creationTimestamp: "2026-07-09T17:53:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:33Z" name: model-chainer-raw-02e34-699f4f88fd-9twms.18c0b096a39142f2 namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: 03d0dc08-4092-496b-9d98-c89a0bf18fcd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-252.ec2.internal source: component: kubelet host: ip-10-0-135-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-02e34} kind: Pod name: model-chainer-raw-02e34-699f4f88fd-9twms namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: a6551cde-8a10-43ca-8c12-cdb8730a9ad6 kind: Event lastTimestamp: "2026-07-09T17:53:35Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:f94a36664a6553b816dc04c45e472a62cdcdb731a5f667f29ab4b59a2331a34e" in 2.303s (2.303s including waiting). Image size: 218072678 bytes.' metadata: creationTimestamp: "2026-07-09T17:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:35Z" name: model-chainer-raw-02e34-699f4f88fd-9twms.18c0b0972ce231ff namespace: kserve-ci-e2e-test resourceVersion: "16547" uid: 83b1d2c0-53b9-4c46-a8d1-93c03f32154e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-252.ec2.internal source: component: kubelet host: ip-10-0-135-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-02e34} kind: Pod name: model-chainer-raw-02e34-699f4f88fd-9twms namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: a6551cde-8a10-43ca-8c12-cdb8730a9ad6 kind: Event lastTimestamp: "2026-07-09T17:53:35Z" message: 'Created container: model-chainer-raw-02e34' metadata: creationTimestamp: "2026-07-09T17:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:35Z" name: model-chainer-raw-02e34-699f4f88fd-9twms.18c0b0973014eace namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: 66ecc78c-6989-422c-b063-2b3bba5076a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-252.ec2.internal source: component: kubelet host: ip-10-0-135-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-02e34} kind: Pod name: model-chainer-raw-02e34-699f4f88fd-9twms namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: a6551cde-8a10-43ca-8c12-cdb8730a9ad6 kind: Event lastTimestamp: "2026-07-09T17:53:35Z" message: Started container model-chainer-raw-02e34 metadata: creationTimestamp: "2026-07-09T17:53:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:35Z" name: model-chainer-raw-02e34-699f4f88fd-9twms.18c0b097308c2b38 namespace: kserve-ci-e2e-test resourceVersion: "16551" uid: d2e8ab35-ed7f-4e1a-88de-ba8e04ff2ee2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-252.ec2.internal source: component: kubelet host: ip-10-0-135-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-02e34} kind: Pod name: model-chainer-raw-02e34-699f4f88fd-9twms namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: a6551cde-8a10-43ca-8c12-cdb8730a9ad6 kind: Event lastTimestamp: "2026-07-09T17:53:42Z" message: Stopping container model-chainer-raw-02e34 metadata: creationTimestamp: "2026-07-09T17:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:53:42Z" name: model-chainer-raw-02e34-699f4f88fd-9twms.18c0b098bbf6e53b namespace: kserve-ci-e2e-test resourceVersion: "16620" uid: 1528e0c4-e473-4744-a3e3-2e03ad76a9f2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-252.ec2.internal source: component: kubelet host: ip-10-0-135-252.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T17:53:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-02e34} kind: Pod name: model-chainer-raw-02e34-699f4f88fd-9twms namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: a6551cde-8a10-43ca-8c12-cdb8730a9ad6 kind: Event lastTimestamp: "2026-07-09T17:54:07Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T17:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:54:07Z" name: model-chainer-raw-02e34-699f4f88fd-9twms.18c0b098bc5642da namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: 1aba160a-d5b9-4631-8af6-920f056fb428 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-252.ec2.internal source: component: kubelet host: ip-10-0-135-252.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:54:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-02e34} kind: Pod name: model-chainer-raw-02e34-699f4f88fd-9twms namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: a6551cde-8a10-43ca-8c12-cdb8730a9ad6 kind: Event lastTimestamp: "2026-07-09T17:54:12Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8080/readyz": dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T17:54:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:54:12Z" name: model-chainer-raw-02e34-699f4f88fd-9twms.18c0b09fb85d46e6 namespace: kserve-ci-e2e-test resourceVersion: "17003" uid: 05afd994-816a-4c46-abc0-64b3e5f4d7f4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-252.ec2.internal source: component: kubelet host: ip-10-0-135-252.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-02e34-699f4f88fd namespace: kserve-ci-e2e-test resourceVersion: "16501" uid: 44c2ea71-2ab1-47df-b15f-e756c3277ac6 kind: Event lastTimestamp: "2026-07-09T17:53:32Z" message: 'Created pod: model-chainer-raw-02e34-699f4f88fd-9twms' metadata: creationTimestamp: "2026-07-09T17:53:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:53:32Z" name: model-chainer-raw-02e34-699f4f88fd.18c0b096643b9cb3 namespace: kserve-ci-e2e-test resourceVersion: "16507" uid: 7cbcca05-af82-44c3-8324-b8dabca4fb74 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-02e34 namespace: kserve-ci-e2e-test resourceVersion: "16499" uid: 4f0149bf-4585-4e21-be46-1e4326e46d37 kind: Event lastTimestamp: "2026-07-09T17:53:32Z" message: Scaled up replica set model-chainer-raw-02e34-699f4f88fd from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:53:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:53:32Z" name: model-chainer-raw-02e34.18c0b09662de471a namespace: kserve-ci-e2e-test resourceVersion: "16502" uid: 8cda02aa-7341-410b-a8fd-20ba9a5d809a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:53:32Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-02e34 namespace: kserve-ci-e2e-test resourceVersion: "16516" uid: 2c0b3880-4e81-497b-9e73-221224833a9b kind: Event lastTimestamp: "2026-07-09T17:53:32Z" message: InferenceGraph [model-chainer-raw-02e34] is Ready metadata: creationTimestamp: "2026-07-09T17:53:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:53:32Z" name: model-chainer-raw-02e34.18c0b0966840165c namespace: kserve-ci-e2e-test resourceVersion: "16518" uid: 44e283c9-8fc1-4dbf-aa9c-9322b0313892 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T17:55:12.750514Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-079fb-7cb556b7bb-dshl5 namespace: kserve-ci-e2e-test resourceVersion: "17471" uid: 3398e206-8c7a-41d8-9d11-e4964124dbe4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-079fb-7cb556b7bb-dshl5 to ip-10-0-135-252.ec2.internal metadata: creationTimestamp: "2026-07-09T17:55:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T17:55:12Z" name: model-chainer-raw-hpa-079fb-7cb556b7bb-dshl5.18c0b0adc0e11b0c namespace: kserve-ci-e2e-test resourceVersion: "17475" uid: 9a996ea3-0a86-4f9a-8be9-dbd80b70a376 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-74fd97cb5b-77b77 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T17:55:12Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-079fb-7cb556b7bb-dshl5 namespace: kserve-ci-e2e-test resourceVersion: "17473" uid: 3398e206-8c7a-41d8-9d11-e4964124dbe4 kind: Event lastTimestamp: "2026-07-09T17:55:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-079fb-serving-cert" not found' metadata: creationTimestamp: "2026-07-09T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:13Z" name: model-chainer-raw-hpa-079fb-7cb556b7bb-dshl5.18c0b0adcc41f5ec namespace: kserve-ci-e2e-test resourceVersion: "17489" uid: 0094bfcf-4266-4859-a040-159dbb169998 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-252.ec2.internal source: component: kubelet host: ip-10-0-135-252.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:14Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-079fb-7cb556b7bb-dshl5 namespace: kserve-ci-e2e-test resourceVersion: "17479" uid: 3398e206-8c7a-41d8-9d11-e4964124dbe4 kind: Event lastTimestamp: "2026-07-09T17:55:14Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T17:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T17:55:14Z" name: model-chainer-raw-hpa-079fb-7cb556b7bb-dshl5.18c0b0ae33c4fd72 namespace: kserve-ci-e2e-test resourceVersion: "17503" uid: acebbd04-4d02-4930-b2ef-2f305011b843 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-079fb} kind: Pod name: model-chainer-raw-hpa-079fb-7cb556b7bb-dshl5 namespace: kserve-ci-e2e-test resourceVersion: "17473" uid: 3398e206-8c7a-41d8-9d11-e4964124dbe4 kind: Event lastTimestamp: "2026-07-09T17:55:14Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f94a36664a6553b816dc04c45e472a62cdcdb731a5f667f29ab4b59a2331a34e" already present on machine metadata: creationTimestamp: "2026-07-09T17:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:14Z" name: model-chainer-raw-hpa-079fb-7cb556b7bb-dshl5.18c0b0ae3506398d namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: f827d027-d0b0-44b8-a7a1-141e6b86151c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-252.ec2.internal source: component: kubelet host: ip-10-0-135-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-079fb} kind: Pod name: model-chainer-raw-hpa-079fb-7cb556b7bb-dshl5 namespace: kserve-ci-e2e-test resourceVersion: "17473" uid: 3398e206-8c7a-41d8-9d11-e4964124dbe4 kind: Event lastTimestamp: "2026-07-09T17:55:14Z" message: 'Created container: model-chainer-raw-hpa-079fb' metadata: creationTimestamp: "2026-07-09T17:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:14Z" name: model-chainer-raw-hpa-079fb-7cb556b7bb-dshl5.18c0b0ae3811b34b namespace: kserve-ci-e2e-test resourceVersion: "17507" uid: 2d556214-d720-44d4-9e1e-e7278cdc5d33 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-252.ec2.internal source: component: kubelet host: ip-10-0-135-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-079fb} kind: Pod name: model-chainer-raw-hpa-079fb-7cb556b7bb-dshl5 namespace: kserve-ci-e2e-test resourceVersion: "17473" uid: 3398e206-8c7a-41d8-9d11-e4964124dbe4 kind: Event lastTimestamp: "2026-07-09T17:55:14Z" message: Started container model-chainer-raw-hpa-079fb metadata: creationTimestamp: "2026-07-09T17:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:14Z" name: model-chainer-raw-hpa-079fb-7cb556b7bb-dshl5.18c0b0ae3885ce35 namespace: kserve-ci-e2e-test resourceVersion: "17508" uid: 8b7d5f91-ba8b-4285-b655-6683e85a7dcc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-252.ec2.internal source: component: kubelet host: ip-10-0-135-252.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-079fb} kind: Pod name: model-chainer-raw-hpa-079fb-7cb556b7bb-dshl5 namespace: kserve-ci-e2e-test resourceVersion: "17473" uid: 3398e206-8c7a-41d8-9d11-e4964124dbe4 kind: Event lastTimestamp: "2026-07-09T17:55:22Z" message: Stopping container model-chainer-raw-hpa-079fb metadata: creationTimestamp: "2026-07-09T17:55:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:22Z" name: model-chainer-raw-hpa-079fb-7cb556b7bb-dshl5.18c0b0b01c947d7e namespace: kserve-ci-e2e-test resourceVersion: "17590" uid: 8922e4b5-a75f-48d1-921e-63ac9548a2da reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-252.ec2.internal source: component: kubelet host: ip-10-0-135-252.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-09T17:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-079fb} kind: Pod name: model-chainer-raw-hpa-079fb-7cb556b7bb-dshl5 namespace: kserve-ci-e2e-test resourceVersion: "17473" uid: 3398e206-8c7a-41d8-9d11-e4964124dbe4 kind: Event lastTimestamp: "2026-07-09T17:55:51Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-09T17:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T17:55:51Z" name: model-chainer-raw-hpa-079fb-7cb556b7bb-dshl5.18c0b0b0e2393a5a namespace: kserve-ci-e2e-test resourceVersion: "17972" uid: 4f8211f4-0c56-479e-af93-17bc09179746 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-252.ec2.internal source: component: kubelet host: ip-10-0-135-252.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-079fb-7cb556b7bb namespace: kserve-ci-e2e-test resourceVersion: "17465" uid: 5453f04a-1717-4138-8770-0bc27e449681 kind: Event lastTimestamp: "2026-07-09T17:55:12Z" message: 'Created pod: model-chainer-raw-hpa-079fb-7cb556b7bb-dshl5' metadata: creationTimestamp: "2026-07-09T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:55:12Z" name: model-chainer-raw-hpa-079fb-7cb556b7bb.18c0b0adc0575efa namespace: kserve-ci-e2e-test resourceVersion: "17472" uid: 45c07a5e-4b85-43e0-9bef-634ad8748ec6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-079fb namespace: kserve-ci-e2e-test resourceVersion: "17463" uid: 850b0c17-93c0-441a-a43f-d1048c98f9aa kind: Event lastTimestamp: "2026-07-09T17:55:12Z" message: Scaled up replica set model-chainer-raw-hpa-079fb-7cb556b7bb from 0 to 1 metadata: creationTimestamp: "2026-07-09T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T17:55:12Z" name: model-chainer-raw-hpa-079fb.18c0b0adbf131540 namespace: kserve-ci-e2e-test resourceVersion: "17466" uid: e8107605-1be6-4f53-94fc-1c2a4726677c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T17:55:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-079fb namespace: kserve-ci-e2e-test resourceVersion: "17483" uid: ae7f388f-372f-4382-ba2c-377c5ceb5f8c kind: Event lastTimestamp: "2026-07-09T17:55:12Z" message: InferenceGraph [model-chainer-raw-hpa-079fb] is Ready metadata: creationTimestamp: "2026-07-09T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T17:55:12Z" name: model-chainer-raw-hpa-079fb.18c0b0adc47af72d namespace: kserve-ci-e2e-test resourceVersion: "17485" uid: d4527f9c-d4b0-4efe-b12c-652320d59860 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:02:12.488017Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r namespace: kserve-ci-e2e-test resourceVersion: "21085" uid: 87ec386b-a148-4878-8742-6fdb583f9173 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-f062e-predictor-797bb4b66f-2ws4r to ip-10-0-143-142.ec2.internal metadata: creationTimestamp: "2026-07-09T18:02:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:02:12Z" name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r.18c0b10f7b2f20b3 namespace: kserve-ci-e2e-test resourceVersion: "21091" uid: c7b09ab8-c254-47c7-8eb0-79d66e45fbe9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-74fd97cb5b-77b77 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:12Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r namespace: kserve-ci-e2e-test resourceVersion: "21096" uid: 87ec386b-a148-4878-8742-6fdb583f9173 kind: Event lastTimestamp: "2026-07-09T18:02:12Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:02:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:02:12Z" name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r.18c0b10f94a814ac namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: 35763181-299e-4cac-80ca-77abd1c26fb5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 87ec386b-a148-4878-8742-6fdb583f9173 kind: Event lastTimestamp: "2026-07-09T18:02:12Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6455858d9bcb13c4793d39d844d39ea45d90725b82962b5a6791b22458b6ca42" already present on machine metadata: creationTimestamp: "2026-07-09T18:02:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:12Z" name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r.18c0b10f9619aaeb namespace: kserve-ci-e2e-test resourceVersion: "21111" uid: 21a028d3-3b68-4d72-abcb-b9d6622238ff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 87ec386b-a148-4878-8742-6fdb583f9173 kind: Event lastTimestamp: "2026-07-09T18:02:12Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T18:02:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:12Z" name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r.18c0b10f98d29e3f namespace: kserve-ci-e2e-test resourceVersion: "21112" uid: fcb039af-0f39-412b-8dbc-dfd68eee8685 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 87ec386b-a148-4878-8742-6fdb583f9173 kind: Event lastTimestamp: "2026-07-09T18:02:12Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T18:02:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:12Z" name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r.18c0b10f99895334 namespace: kserve-ci-e2e-test resourceVersion: "21113" uid: 7b88c0d3-dc97-4dcb-aecc-8ef88254d2c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 87ec386b-a148-4878-8742-6fdb583f9173 kind: Event lastTimestamp: "2026-07-09T18:02:17Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T18:02:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:17Z" name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r.18c0b11093abb1eb namespace: kserve-ci-e2e-test resourceVersion: "21150" uid: 55c321eb-8e34-40ed-993e-ba5f350a47a5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 87ec386b-a148-4878-8742-6fdb583f9173 kind: Event lastTimestamp: "2026-07-09T18:02:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T18:02:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:17Z" name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r.18c0b11096c2162d namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: d30cf1d1-71bb-485f-b9cd-f3eb5766a110 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 87ec386b-a148-4878-8742-6fdb583f9173 kind: Event lastTimestamp: "2026-07-09T18:02:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T18:02:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:17Z" name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r.18c0b11097316c35 namespace: kserve-ci-e2e-test resourceVersion: "21153" uid: 5172f143-8943-445f-892a-680e32c37944 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 87ec386b-a148-4878-8742-6fdb583f9173 kind: Event lastTimestamp: "2026-07-09T18:02:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T18:02:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:17Z" name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r.18c0b110973aab9c namespace: kserve-ci-e2e-test resourceVersion: "21154" uid: 14e495d0-1c29-4c17-8657-bbbf84259e02 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 87ec386b-a148-4878-8742-6fdb583f9173 kind: Event lastTimestamp: "2026-07-09T18:02:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T18:02:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:17Z" name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r.18c0b11099cc3326 namespace: kserve-ci-e2e-test resourceVersion: "21157" uid: 50eb6b54-4223-4bfb-a896-b2fc0037f77b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 87ec386b-a148-4878-8742-6fdb583f9173 kind: Event lastTimestamp: "2026-07-09T18:02:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:02:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:02:17Z" name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r.18c0b1109a39e4ff namespace: kserve-ci-e2e-test resourceVersion: "21158" uid: 5c5b1e6a-f117-4d83-b9e1-11e905937cb7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-09T18:02:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 87ec386b-a148-4878-8742-6fdb583f9173 kind: Event lastTimestamp: "2026-07-09T18:03:44Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:02:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:03:44Z" name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r.18c0b110cfad0867 namespace: kserve-ci-e2e-test resourceVersion: "21799" uid: d64ba499-eb3c-49b5-a775-a7ff8fe48d42 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 87ec386b-a148-4878-8742-6fdb583f9173 kind: Event lastTimestamp: "2026-07-09T18:03:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T18:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:03:42Z" name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r.18c0b12478ba84b1 namespace: kserve-ci-e2e-test resourceVersion: "21786" uid: 3d8e5e20-0bad-48b8-a4dc-67333849a3d5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 87ec386b-a148-4878-8742-6fdb583f9173 kind: Event lastTimestamp: "2026-07-09T18:03:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:03:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:03:42Z" name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r.18c0b12478bbb44c namespace: kserve-ci-e2e-test resourceVersion: "21787" uid: 86a0a2f1-9a74-4f62-b250-630b81bef150 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 87ec386b-a148-4878-8742-6fdb583f9173 kind: Event lastTimestamp: "2026-07-09T18:03:44Z" message: 'Readiness probe failed: Get "https://10.132.0.34:8643/healthz": dial tcp 10.132.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:03:44Z" name: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r.18c0b124d5d3696e namespace: kserve-ci-e2e-test resourceVersion: "21798" uid: 5ff8bb57-8f45-49e7-b107-ba23888cf1b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-f062e-predictor-797bb4b66f namespace: kserve-ci-e2e-test resourceVersion: "21077" uid: 00d6a07b-c912-4a8b-9724-a2babad3e838 kind: Event lastTimestamp: "2026-07-09T18:02:12Z" message: 'Created pod: raw-sklearn-f062e-predictor-797bb4b66f-2ws4r' metadata: creationTimestamp: "2026-07-09T18:02:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:02:12Z" name: raw-sklearn-f062e-predictor-797bb4b66f.18c0b10f7ad1d454 namespace: kserve-ci-e2e-test resourceVersion: "21088" uid: e24ae800-f0a0-47c5-8d97-096bc17e0490 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-f062e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21075" uid: 2ac4a50c-050a-4729-86b1-c21dc2218d1f kind: Event lastTimestamp: "2026-07-09T18:02:12Z" message: Scaled up replica set raw-sklearn-f062e-predictor-797bb4b66f from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:02:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:02:12Z" name: raw-sklearn-f062e-predictor.18c0b10f77bfc0d6 namespace: kserve-ci-e2e-test resourceVersion: "21079" uid: c5cb3883-3d6e-402e-879d-d9cbffdaf544 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:02:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-f062e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21076" uid: 384ff5b7-7497-4d7c-bf5d-f2fbccfbff44 kind: Event lastTimestamp: "2026-07-09T18:02:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:02:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:02:42Z" name: raw-sklearn-f062e-predictor.18c0b112f65fbbe2 namespace: kserve-ci-e2e-test resourceVersion: "21340" uid: 10615fa9-7abe-47ab-bdf8-4a4f3da5a38d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:02:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-f062e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21076" uid: 384ff5b7-7497-4d7c-bf5d-f2fbccfbff44 kind: Event lastTimestamp: "2026-07-09T18:02:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:02:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:02:42Z" name: raw-sklearn-f062e-predictor.18c0b112f6c4ee3f namespace: kserve-ci-e2e-test resourceVersion: "21341" uid: e2373e26-4d67-4cde-aa5d-fa9ca95869b7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T18:02:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-f062e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21237" uid: 384ff5b7-7497-4d7c-bf5d-f2fbccfbff44 kind: Event lastTimestamp: "2026-07-09T18:03:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:02:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:03:42Z" name: raw-sklearn-f062e-predictor.18c0b119f4851b02 namespace: kserve-ci-e2e-test resourceVersion: "21769" uid: d22dcfd3-bce9-4bc8-af1a-fced5451a092 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T18:02:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-f062e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21237" uid: 384ff5b7-7497-4d7c-bf5d-f2fbccfbff44 kind: Event lastTimestamp: "2026-07-09T18:03:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:02:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:03:42Z" name: raw-sklearn-f062e-predictor.18c0b119f4ea85e7 namespace: kserve-ci-e2e-test resourceVersion: "21770" uid: f9c2d3b4-9291-494e-945d-048ffb0eedd0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-07-09T18:02:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-f062e namespace: kserve-ci-e2e-test resourceVersion: "21067" uid: c45deb88-b0f8-4ba4-b277-977f70ac45f4 kind: Event lastTimestamp: "2026-07-09T18:03:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T18:02:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:03:34Z" name: raw-sklearn-f062e.18c0b10f77bdca8a namespace: kserve-ci-e2e-test resourceVersion: "21710" uid: 49d43d08-82cb-4a30-8e85-374ff3a54abc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:02:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-f062e namespace: kserve-ci-e2e-test resourceVersion: "21067" uid: c45deb88-b0f8-4ba4-b277-977f70ac45f4 kind: Event lastTimestamp: "2026-07-09T18:02:12Z" message: 'Failed to update status for InferenceService "raw-sklearn-f062e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-f062e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:02:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:02:12Z" name: raw-sklearn-f062e.18c0b10f7841e75b namespace: kserve-ci-e2e-test resourceVersion: "21098" uid: 00efb0e3-26b9-4b59-9b4d-a49821641502 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:02:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-f062e namespace: kserve-ci-e2e-test resourceVersion: "21070" uid: c45deb88-b0f8-4ba4-b277-977f70ac45f4 kind: Event lastTimestamp: "2026-07-09T18:02:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-f062e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:02:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:02:12Z" name: raw-sklearn-f062e.18c0b10f7e3edfcc namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: ddcd55eb-5356-4ac5-89f4-a5296ff0bd39 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-f062e namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: c45deb88-b0f8-4ba4-b277-977f70ac45f4 kind: Event lastTimestamp: "2026-07-09T18:03:34Z" message: InferenceService [raw-sklearn-f062e] is Ready metadata: creationTimestamp: "2026-07-09T18:03:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:03:34Z" name: raw-sklearn-f062e.18c0b122871091b1 namespace: kserve-ci-e2e-test resourceVersion: "21708" uid: 58694038-f25b-4b0b-b23d-9b801c9e1aa1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-09T18:03:47.895450Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4 namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: 0ad0bc00-653f-4dac-ac58-689d70570066 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4 to ip-10-0-143-142.ec2.internal metadata: creationTimestamp: "2026-07-09T18:03:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-09T18:03:47Z" name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4.18c0b125b1e8f787 namespace: kserve-ci-e2e-test resourceVersion: "21857" uid: a7fb1a3f-c2c4-4cf3-bfb5-b92830cdf1db reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-74fd97cb5b-77b77 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:48Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4 namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: 0ad0bc00-653f-4dac-ac58-689d70570066 kind: Event lastTimestamp: "2026-07-09T18:03:48Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-09T18:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-09T18:03:48Z" name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4.18c0b125cb6e7178 namespace: kserve-ci-e2e-test resourceVersion: "21871" uid: dcf84062-357f-42d7-b777-508e1a75fa5b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4 namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 0ad0bc00-653f-4dac-ac58-689d70570066 kind: Event lastTimestamp: "2026-07-09T18:03:48Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6455858d9bcb13c4793d39d844d39ea45d90725b82962b5a6791b22458b6ca42" already present on machine metadata: creationTimestamp: "2026-07-09T18:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:03:48Z" name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4.18c0b125cd06faa2 namespace: kserve-ci-e2e-test resourceVersion: "21873" uid: 58ad299d-6781-4254-bde1-466f304eb8ab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4 namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 0ad0bc00-653f-4dac-ac58-689d70570066 kind: Event lastTimestamp: "2026-07-09T18:03:48Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-09T18:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:03:48Z" name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4.18c0b125cfb06baa namespace: kserve-ci-e2e-test resourceVersion: "21874" uid: b32c910b-50cd-4a9e-acb1-f7d768e8cfa1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4 namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 0ad0bc00-653f-4dac-ac58-689d70570066 kind: Event lastTimestamp: "2026-07-09T18:03:48Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-09T18:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:03:48Z" name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4.18c0b125d02459d0 namespace: kserve-ci-e2e-test resourceVersion: "21875" uid: 1f857747-1ad5-49f7-8164-4786ed62ceb1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4 namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 0ad0bc00-653f-4dac-ac58-689d70570066 kind: Event lastTimestamp: "2026-07-09T18:03:52Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1714" already present on machine metadata: creationTimestamp: "2026-07-09T18:03:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:03:52Z" name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4.18c0b126c8f68c3e namespace: kserve-ci-e2e-test resourceVersion: "21908" uid: 98f440b3-0cb7-4004-b6ba-87a84821d0f3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4 namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 0ad0bc00-653f-4dac-ac58-689d70570066 kind: Event lastTimestamp: "2026-07-09T18:03:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-09T18:03:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:03:52Z" name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4.18c0b126cc657904 namespace: kserve-ci-e2e-test resourceVersion: "21911" uid: 566606c8-23d5-463d-a87c-1d445370bcf2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4 namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 0ad0bc00-653f-4dac-ac58-689d70570066 kind: Event lastTimestamp: "2026-07-09T18:03:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-09T18:03:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:03:52Z" name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4.18c0b126ccd85c5c namespace: kserve-ci-e2e-test resourceVersion: "21913" uid: 86477a9a-1c8e-49be-9bc0-6df4ec45cf78 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4 namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 0ad0bc00-653f-4dac-ac58-689d70570066 kind: Event lastTimestamp: "2026-07-09T18:03:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-09T18:03:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:03:52Z" name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4.18c0b126cce270c0 namespace: kserve-ci-e2e-test resourceVersion: "21914" uid: 077b88df-746b-4248-b5f0-29648e2a55ac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4 namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 0ad0bc00-653f-4dac-ac58-689d70570066 kind: Event lastTimestamp: "2026-07-09T18:03:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-09T18:03:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:03:52Z" name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4.18c0b126d01fded8 namespace: kserve-ci-e2e-test resourceVersion: "21917" uid: 322828f6-9719-4e28-9c56-8e9812148811 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4 namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 0ad0bc00-653f-4dac-ac58-689d70570066 kind: Event lastTimestamp: "2026-07-09T18:03:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:03:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:03:52Z" name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4.18c0b126d0922062 namespace: kserve-ci-e2e-test resourceVersion: "21918" uid: e9646799-99df-4f88-b410-e1d740a07bf2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-09T18:03:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4 namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 0ad0bc00-653f-4dac-ac58-689d70570066 kind: Event lastTimestamp: "2026-07-09T18:05:20Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:03:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:05:20Z" name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4.18c0b12740c3d211 namespace: kserve-ci-e2e-test resourceVersion: "22572" uid: e356b326-1b66-41db-b189-8d1ee21e4bba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:05:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4 namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 0ad0bc00-653f-4dac-ac58-689d70570066 kind: Event lastTimestamp: "2026-07-09T18:05:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-09T18:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:05:17Z" name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4.18c0b13aa974b376 namespace: kserve-ci-e2e-test resourceVersion: "22547" uid: 178e62b7-7a1e-4a7d-9ea1-7826a4c6d228 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:05:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4 namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 0ad0bc00-653f-4dac-ac58-689d70570066 kind: Event lastTimestamp: "2026-07-09T18:05:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-09T18:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:05:17Z" name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4.18c0b13aa975594b namespace: kserve-ci-e2e-test resourceVersion: "22548" uid: 22078e23-3ad0-4445-bda2-435673eff209 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:05:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4 namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 0ad0bc00-653f-4dac-ac58-689d70570066 kind: Event lastTimestamp: "2026-07-09T18:05:20Z" message: 'Readiness probe failed: Get "https://10.132.0.35:8643/healthz": dial tcp 10.132.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-09T18:05:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-09T18:05:20Z" name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4.18c0b13b4700d2af namespace: kserve-ci-e2e-test resourceVersion: "22571" uid: f50b3568-8fad-428a-9afd-6261e0f202f7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-142.ec2.internal source: component: kubelet host: ip-10-0-143-142.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9 namespace: kserve-ci-e2e-test resourceVersion: "21841" uid: d25d1f69-19c2-423c-ad87-77e8acf8e6d9 kind: Event lastTimestamp: "2026-07-09T18:03:47Z" message: 'Created pod: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9-mm9h4' metadata: creationTimestamp: "2026-07-09T18:03:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:03:47Z" name: raw-sklearn-runtime-4d85b-predictor-5846b9b6b9.18c0b125b183191f namespace: kserve-ci-e2e-test resourceVersion: "21853" uid: 1f674146-c97c-4422-ba6b-9313770faa32 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-4d85b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21839" uid: 5751cb82-f15e-42e6-9b5c-4a3be88951d2 kind: Event lastTimestamp: "2026-07-09T18:03:47Z" message: Scaled up replica set raw-sklearn-runtime-4d85b-predictor-5846b9b6b9 from 0 to 1 metadata: creationTimestamp: "2026-07-09T18:03:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:03:47Z" name: raw-sklearn-runtime-4d85b-predictor.18c0b125ad5c0fc1 namespace: kserve-ci-e2e-test resourceVersion: "21843" uid: f947f301-e0a4-4a90-aa62-72db85346f31 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:04:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-4d85b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: 200d77cd-1d43-441e-9c91-b9348fe0e790 kind: Event lastTimestamp: "2026-07-09T18:04:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:04:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:04:17Z" name: raw-sklearn-runtime-4d85b-predictor.18c0b1292baa439d namespace: kserve-ci-e2e-test resourceVersion: "22099" uid: 116c67cf-0bb8-46ad-9227-9e38b9db1b8c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-09T18:04:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-4d85b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: 200d77cd-1d43-441e-9c91-b9348fe0e790 kind: Event lastTimestamp: "2026-07-09T18:04:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-09T18:04:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:04:17Z" name: raw-sklearn-runtime-4d85b-predictor.18c0b1292c1c9424 namespace: kserve-ci-e2e-test resourceVersion: "22100" uid: ad3e20e5-4335-4a95-a916-c58222172a8b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T18:04:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-4d85b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21994" uid: 200d77cd-1d43-441e-9c91-b9348fe0e790 kind: Event lastTimestamp: "2026-07-09T18:05:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:05:17Z" name: raw-sklearn-runtime-4d85b-predictor.18c0b13029df1a14 namespace: kserve-ci-e2e-test resourceVersion: "22542" uid: 4aca20b1-387c-4934-816e-f99555d75455 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-09T18:04:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-4d85b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21994" uid: 200d77cd-1d43-441e-9c91-b9348fe0e790 kind: Event lastTimestamp: "2026-07-09T18:05:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-09T18:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-09T18:05:17Z" name: raw-sklearn-runtime-4d85b-predictor.18c0b1302a751ca0 namespace: kserve-ci-e2e-test resourceVersion: "22543" uid: 5547ca3d-95c8-478d-8d2b-c7e13b448ea0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-09T18:03:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-4d85b namespace: kserve-ci-e2e-test resourceVersion: "21830" uid: 9f9d8ba4-8b15-456b-ade4-0705112ecff5 kind: Event lastTimestamp: "2026-07-09T18:05:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-09T18:03:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:05:10Z" name: raw-sklearn-runtime-4d85b.18c0b125ad5e1deb namespace: kserve-ci-e2e-test resourceVersion: "22479" uid: f0017009-9f0c-497b-aa9f-5449a73bda18 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:03:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-4d85b namespace: kserve-ci-e2e-test resourceVersion: "21830" uid: 9f9d8ba4-8b15-456b-ade4-0705112ecff5 kind: Event lastTimestamp: "2026-07-09T18:03:47Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-4d85b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-4d85b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-09T18:03:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:03:47Z" name: raw-sklearn-runtime-4d85b.18c0b125adcddfa3 namespace: kserve-ci-e2e-test resourceVersion: "21845" uid: db58ed81-c36f-49e3-bed6-99c90cb91717 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-09T18:05:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-4d85b namespace: kserve-ci-e2e-test resourceVersion: "22477" uid: 9f9d8ba4-8b15-456b-ade4-0705112ecff5 kind: Event lastTimestamp: "2026-07-09T18:05:10Z" message: InferenceService [raw-sklearn-runtime-4d85b] is Ready metadata: creationTimestamp: "2026-07-09T18:05:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-09T18:05:10Z" name: raw-sklearn-runtime-4d85b.18c0b138f986ad33 namespace: kserve-ci-e2e-test resourceVersion: "22478" uid: 78461053-5512-4f12-a980-afa3bf063f40 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22809"