--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:30:41.519328Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-da7d24-predictor-65cb8489c4-zxr7f namespace: kserve-ci-e2e-test resourceVersion: "20306" uid: 7c0b28be-2ecd-4099-be11-6381321e2562 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-da7d24-predictor-65cb8489c4-zxr7f to ip-10-0-140-100.ec2.internal metadata: creationTimestamp: "2026-07-07T14:30:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:30:41Z" name: isvc-init-fail-da7d24-predictor-65cb8489c4-zxr7f.18c0085b7fe2d564 namespace: kserve-ci-e2e-test resourceVersion: "20312" uid: 03575bfe-5adc-4b35-93b4-dd5ce20459d2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84448d6d9-ddnp2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:41Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-da7d24-predictor-65cb8489c4-zxr7f namespace: kserve-ci-e2e-test resourceVersion: "20318" uid: 7c0b28be-2ecd-4099-be11-6381321e2562 kind: Event lastTimestamp: "2026-07-07T14:30:41Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:30:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:30:41Z" name: isvc-init-fail-da7d24-predictor-65cb8489c4-zxr7f.18c0085b994677f9 namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: 512070cc-3d0a-4136-bf12-7715ff2ce3a5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:30:41Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-da7d24-predictor-65cb8489c4-zxr7f namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 7c0b28be-2ecd-4099-be11-6381321e2562 kind: Event lastTimestamp: "2026-07-07T14:30:47Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:30:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:30:47Z" name: isvc-init-fail-da7d24-predictor-65cb8489c4-zxr7f.18c0085b9acf2406 namespace: kserve-ci-e2e-test resourceVersion: "20370" uid: 6e942eba-4c3f-432a-9518-9d5b3fd9e62d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:30:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-da7d24-predictor-65cb8489c4-zxr7f namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 7c0b28be-2ecd-4099-be11-6381321e2562 kind: Event lastTimestamp: "2026-07-07T14:30:47Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:30:47Z" name: isvc-init-fail-da7d24-predictor-65cb8489c4-zxr7f.18c0085b9dcb7591 namespace: kserve-ci-e2e-test resourceVersion: "20372" uid: 572d1c56-1a63-44c9-b5fe-7d7af350f715 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:30:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-da7d24-predictor-65cb8489c4-zxr7f namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 7c0b28be-2ecd-4099-be11-6381321e2562 kind: Event lastTimestamp: "2026-07-07T14:30:47Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:30:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:30:47Z" name: isvc-init-fail-da7d24-predictor-65cb8489c4-zxr7f.18c0085b9e3ab3f0 namespace: kserve-ci-e2e-test resourceVersion: "20373" uid: eaa59c02-bca2-44b6-b485-2b74de340459 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-da7d24-predictor-65cb8489c4-zxr7f namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 7c0b28be-2ecd-4099-be11-6381321e2562 kind: Event lastTimestamp: "2026-07-07T14:30:51Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-07-07T14:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:30:51Z" name: isvc-init-fail-da7d24-predictor-65cb8489c4-zxr7f.18c0085dd33f1e68 namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: 91dac939-9649-48f5-b522-bef996e9cef5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-da7d24-predictor-65cb8489c4 namespace: kserve-ci-e2e-test resourceVersion: "20298" uid: 47a65d2f-2a08-4157-868d-8cd5813870cc kind: Event lastTimestamp: "2026-07-07T14:30:41Z" message: 'Created pod: isvc-init-fail-da7d24-predictor-65cb8489c4-zxr7f' metadata: creationTimestamp: "2026-07-07T14:30:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:30:41Z" name: isvc-init-fail-da7d24-predictor-65cb8489c4.18c0085b7f6c76e1 namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: be32483e-ece8-4da2-9951-4b796bed3a5b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-da7d24-predictor namespace: kserve-ci-e2e-test resourceVersion: "20296" uid: de18d961-4482-4e68-96ea-5c6c3c879fa6 kind: Event lastTimestamp: "2026-07-07T14:30:41Z" message: Scaled up replica set isvc-init-fail-da7d24-predictor-65cb8489c4 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:30:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:30:41Z" name: isvc-init-fail-da7d24-predictor.18c0085b7c708c39 namespace: kserve-ci-e2e-test resourceVersion: "20300" uid: afcd7508-e947-40cb-ba8f-fb4d896822b8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-07T14:30:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-da7d24 namespace: kserve-ci-e2e-test resourceVersion: "20289" uid: 1058b231-8502-482b-9bd8-9d9d863b2fa1 kind: Event lastTimestamp: "2026-07-07T14:30:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:30:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:30:47Z" name: isvc-init-fail-da7d24.18c0085b7c5060a5 namespace: kserve-ci-e2e-test resourceVersion: "20377" uid: b1b07ee4-d4e2-460e-82a3-927bf9b00e80 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-da7d24 namespace: kserve-ci-e2e-test resourceVersion: "20289" uid: 1058b231-8502-482b-9bd8-9d9d863b2fa1 kind: Event lastTimestamp: "2026-07-07T14:30:41Z" message: 'Failed to update status for InferenceService "isvc-init-fail-da7d24": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-da7d24": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:30:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:30:41Z" name: isvc-init-fail-da7d24.18c0085b7ca2ee7f namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: f65c5a07-5c26-46ef-a6ee-5e6ebb16e54e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-da7d24 namespace: kserve-ci-e2e-test resourceVersion: "20289" uid: 1058b231-8502-482b-9bd8-9d9d863b2fa1 kind: Event lastTimestamp: "2026-07-07T14:30:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-da7d24": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:30:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:30:41Z" name: isvc-init-fail-da7d24.18c0085b7ca33960 namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: d45ec87c-1299-4161-994b-394f7346e167 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:24:29.512269Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 namespace: kserve-ci-e2e-test resourceVersion: "17284" uid: c8fc836e-1216-4e91-9ef9-19c2043f8178 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 to ip-10-0-140-100.ec2.internal metadata: creationTimestamp: "2026-07-07T14:24:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:24:29Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5.18c00804e2899ae4 namespace: kserve-ci-e2e-test resourceVersion: "17288" uid: 296109c7-4916-4205-af56-ddaf40b2c212 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84448d6d9-ddnp2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:29Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 namespace: kserve-ci-e2e-test resourceVersion: "17293" uid: c8fc836e-1216-4e91-9ef9-19c2043f8178 kind: Event lastTimestamp: "2026-07-07T14:24:29Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:24:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:24:29Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5.18c00804fbb512fe namespace: kserve-ci-e2e-test resourceVersion: "17300" uid: ae29f3c0-063e-4704-8783-1a29290c18cb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: c8fc836e-1216-4e91-9ef9-19c2043f8178 kind: Event lastTimestamp: "2026-07-07T14:24:29Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:24:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:29Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5.18c00804fd49d5fe namespace: kserve-ci-e2e-test resourceVersion: "17302" uid: 8ad145d1-690a-49c8-8321-c4b68caa491f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: c8fc836e-1216-4e91-9ef9-19c2043f8178 kind: Event lastTimestamp: "2026-07-07T14:24:30Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:30Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5.18c00804ffc90b3b namespace: kserve-ci-e2e-test resourceVersion: "17303" uid: af0753fb-0ad3-4ccd-bdda-e3efffaaf779 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: c8fc836e-1216-4e91-9ef9-19c2043f8178 kind: Event lastTimestamp: "2026-07-07T14:24:30Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:24:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:30Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5.18c0080500553b42 namespace: kserve-ci-e2e-test resourceVersion: "17304" uid: 8315b0c9-7735-43d8-b7ed-521e882b09b9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: c8fc836e-1216-4e91-9ef9-19c2043f8178 kind: Event lastTimestamp: "2026-07-07T14:24:34Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:24:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:34Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5.18c00806169e5291 namespace: kserve-ci-e2e-test resourceVersion: "17342" uid: 4cb1dfca-04ab-437e-a91e-9fefdd779dde reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: c8fc836e-1216-4e91-9ef9-19c2043f8178 kind: Event lastTimestamp: "2026-07-07T14:24:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:24:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:34Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5.18c008061998d425 namespace: kserve-ci-e2e-test resourceVersion: "17345" uid: e9b460f7-d4fd-40bc-b62b-ac41af5cd13a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: c8fc836e-1216-4e91-9ef9-19c2043f8178 kind: Event lastTimestamp: "2026-07-07T14:24:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:24:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:34Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5.18c008061a05df59 namespace: kserve-ci-e2e-test resourceVersion: "17346" uid: aa52680f-3301-40a8-bfdd-e034c100b643 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: c8fc836e-1216-4e91-9ef9-19c2043f8178 kind: Event lastTimestamp: "2026-07-07T14:24:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:24:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:34Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5.18c008061a0f1bc8 namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: 1773f25f-8552-4ae7-807c-44875434de72 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: c8fc836e-1216-4e91-9ef9-19c2043f8178 kind: Event lastTimestamp: "2026-07-07T14:24:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:24:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:34Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5.18c008061d15507a namespace: kserve-ci-e2e-test resourceVersion: "17350" uid: 030cada0-c01f-4409-885e-d4d670626a46 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: c8fc836e-1216-4e91-9ef9-19c2043f8178 kind: Event lastTimestamp: "2026-07-07T14:24:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:24:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:34Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5.18c008061d84d75f namespace: kserve-ci-e2e-test resourceVersion: "17351" uid: 3482d1b3-93e6-4744-b0d1-2da610d24169 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: c8fc836e-1216-4e91-9ef9-19c2043f8178 kind: Event lastTimestamp: "2026-07-07T14:24:34Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:0b6de7012f6344e0be5c95ac8deb96d56b96edd624ee25e9ad8ecbad40286f86" already present on machine metadata: creationTimestamp: "2026-07-07T14:24:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:34Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5.18c008061d8f8f3b namespace: kserve-ci-e2e-test resourceVersion: "17352" uid: 143a0be2-62ff-4990-8d68-c8fd8d6f26ad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: c8fc836e-1216-4e91-9ef9-19c2043f8178 kind: Event lastTimestamp: "2026-07-07T14:24:34Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-07T14:24:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:34Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5.18c008062068c084 namespace: kserve-ci-e2e-test resourceVersion: "17354" uid: 9c35705a-0143-4038-9954-d8989d1bd75e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: c8fc836e-1216-4e91-9ef9-19c2043f8178 kind: Event lastTimestamp: "2026-07-07T14:24:34Z" message: Started container agent metadata: creationTimestamp: "2026-07-07T14:24:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:34Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5.18c0080620d686de namespace: kserve-ci-e2e-test resourceVersion: "17355" uid: 579c69c9-d2de-4f2f-a3ba-bf5d4a4e52ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T14:24:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: c8fc836e-1216-4e91-9ef9-19c2043f8178 kind: Event lastTimestamp: "2026-07-07T14:26:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:24:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:12Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5.18c0080652a1de4f namespace: kserve-ci-e2e-test resourceVersion: "18177" uid: ab73717f-fecd-46d5-9734-9e63e4b52fea reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T14:24:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: c8fc836e-1216-4e91-9ef9-19c2043f8178 kind: Event lastTimestamp: "2026-07-07T14:26:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T14:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:12Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5.18c008068e818ea4 namespace: kserve-ci-e2e-test resourceVersion: "18178" uid: 9034cabe-54b5-4dca-8025-1a5ac7f83853 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: c8fc836e-1216-4e91-9ef9-19c2043f8178 kind: Event lastTimestamp: "2026-07-07T14:26:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:26:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:04Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5.18c0081b1b8f2810 namespace: kserve-ci-e2e-test resourceVersion: "18058" uid: 82843c5a-536a-4aa4-9069-9d5906222a2a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: c8fc836e-1216-4e91-9ef9-19c2043f8178 kind: Event lastTimestamp: "2026-07-07T14:26:04Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-07T14:26:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:04Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5.18c0081b1b8f700d namespace: kserve-ci-e2e-test resourceVersion: "18065" uid: a432429b-4ef9-4646-a77b-aa992f713ade reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: c8fc836e-1216-4e91-9ef9-19c2043f8178 kind: Event lastTimestamp: "2026-07-07T14:26:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:26:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:04Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5.18c0081b1b8fd66d namespace: kserve-ci-e2e-test resourceVersion: "18069" uid: 510d4335-babb-4c0d-81ea-5359301e45f4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:26:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5 namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: c8fc836e-1216-4e91-9ef9-19c2043f8178 kind: Event lastTimestamp: "2026-07-07T14:26:22Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:26:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:22Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5.18c0081bbeb93059 namespace: kserve-ci-e2e-test resourceVersion: "18252" uid: 6f2bf44c-ad9d-49b1-bd97-c4bf1e604093 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-7a71c-predictor-9798d7c8b namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 2496a7dc-5cbe-47cc-a79a-dbcc93d0400d kind: Event lastTimestamp: "2026-07-07T14:24:29Z" message: 'Created pod: isvc-logger-raw-7a71c-predictor-9798d7c8b-df5l5' metadata: creationTimestamp: "2026-07-07T14:24:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:29Z" name: isvc-logger-raw-7a71c-predictor-9798d7c8b.18c00804e227688c namespace: kserve-ci-e2e-test resourceVersion: "17286" uid: cbe764fa-2bea-46ee-aac0-44d87e9d522f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-7a71c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17270" uid: c6421910-ffe9-4820-9b5a-fecffdde53d2 kind: Event lastTimestamp: "2026-07-07T14:24:29Z" message: Scaled up replica set isvc-logger-raw-7a71c-predictor-9798d7c8b from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:24:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:29Z" name: isvc-logger-raw-7a71c-predictor.18c00804ddefeb88 namespace: kserve-ci-e2e-test resourceVersion: "17275" uid: 9c8fc3c3-4199-4bde-a913-bbc6f17ea3ec reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:24:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-7a71c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17271" uid: da268972-8108-4a6a-aba5-e09dc05ad434 kind: Event lastTimestamp: "2026-07-07T14:24:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:24:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:59Z" name: isvc-logger-raw-7a71c-predictor.18c008085c65189b namespace: kserve-ci-e2e-test resourceVersion: "17549" uid: 9a8290f4-9712-4572-bd00-80e0027889ce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:24:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-7a71c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17271" uid: da268972-8108-4a6a-aba5-e09dc05ad434 kind: Event lastTimestamp: "2026-07-07T14:24:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:24:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:59Z" name: isvc-logger-raw-7a71c-predictor.18c008085cd71ecb namespace: kserve-ci-e2e-test resourceVersion: "17550" uid: 5fc2c51a-91c0-4b09-ab88-f3ab30dce82d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:25:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-7a71c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17438" uid: da268972-8108-4a6a-aba5-e09dc05ad434 kind: Event lastTimestamp: "2026-07-07T14:25:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:59Z" name: isvc-logger-raw-7a71c-predictor.18c0080f5a5c4cf8 namespace: kserve-ci-e2e-test resourceVersion: "17981" uid: 66206287-fceb-4ab0-9a58-3e5703d323c8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:25:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-7a71c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17438" uid: da268972-8108-4a6a-aba5-e09dc05ad434 kind: Event lastTimestamp: "2026-07-07T14:25:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:25:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:25:59Z" name: isvc-logger-raw-7a71c-predictor.18c0080f5ad5fef5 namespace: kserve-ci-e2e-test resourceVersion: "17982" uid: ba1a43cd-91df-43cd-8ab9-10822e00e2cf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-07T14:24:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-7a71c namespace: kserve-ci-e2e-test resourceVersion: "17263" uid: 6234141f-ca3d-4d89-93db-21e21be4878f kind: Event lastTimestamp: "2026-07-07T14:25:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:24:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:25:52Z" name: isvc-logger-raw-7a71c.18c00804ddd5b7a4 namespace: kserve-ci-e2e-test resourceVersion: "17935" uid: 974e7adb-bee2-4d23-9d22-ecd594c1fbb0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-7a71c namespace: kserve-ci-e2e-test resourceVersion: "17263" uid: 6234141f-ca3d-4d89-93db-21e21be4878f kind: Event lastTimestamp: "2026-07-07T14:24:29Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-7a71c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-7a71c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:24:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:29Z" name: isvc-logger-raw-7a71c.18c00804de89712a namespace: kserve-ci-e2e-test resourceVersion: "17277" uid: ab6f818a-405a-4310-9bef-4002e786ab4e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:25:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-7a71c namespace: kserve-ci-e2e-test resourceVersion: "17933" uid: 6234141f-ca3d-4d89-93db-21e21be4878f kind: Event lastTimestamp: "2026-07-07T14:25:52Z" message: InferenceService [isvc-logger-raw-7a71c] is Ready metadata: creationTimestamp: "2026-07-07T14:25:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:25:52Z" name: isvc-logger-raw-7a71c.18c0081846af43da namespace: kserve-ci-e2e-test resourceVersion: "17934" uid: bd79bf19-27a9-45fe-92d9-9d06bbce757e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:28:50.096258Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg namespace: kserve-ci-e2e-test resourceVersion: "19345" uid: 37620da4-15da-4d0d-ad8f-e404d3b8e339 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-07975f-predictor-8574dbdbd7-fvkgg to ip-10-0-140-100.ec2.internal metadata: creationTimestamp: "2026-07-07T14:28:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:28:50Z" name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg.18c008418e8dbc50 namespace: kserve-ci-e2e-test resourceVersion: "19349" uid: 577ac27d-fe06-491d-83fd-b036e3bedf30 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84448d6d9-ddnp2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg namespace: kserve-ci-e2e-test resourceVersion: "19354" uid: 37620da4-15da-4d0d-ad8f-e404d3b8e339 kind: Event lastTimestamp: "2026-07-07T14:28:50Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:28:50Z" name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg.18c00841a7d20aa9 namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: e5160746-8ada-478f-b6e9-58d3afe3406c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 37620da4-15da-4d0d-ad8f-e404d3b8e339 kind: Event lastTimestamp: "2026-07-07T14:28:50Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:28:50Z" name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg.18c00841a946fae1 namespace: kserve-ci-e2e-test resourceVersion: "19364" uid: d015ff2e-3315-4f28-9967-d47e6013f8c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 37620da4-15da-4d0d-ad8f-e404d3b8e339 kind: Event lastTimestamp: "2026-07-07T14:28:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:28:50Z" name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg.18c00841ac2990ed namespace: kserve-ci-e2e-test resourceVersion: "19366" uid: 4ec03be6-6b76-4c48-b774-3112a5d01bf2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 37620da4-15da-4d0d-ad8f-e404d3b8e339 kind: Event lastTimestamp: "2026-07-07T14:28:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:28:50Z" name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg.18c00841ac98920b namespace: kserve-ci-e2e-test resourceVersion: "19367" uid: b76f3d20-a006-453b-88dc-165ef8d071ca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 37620da4-15da-4d0d-ad8f-e404d3b8e339 kind: Event lastTimestamp: "2026-07-07T14:28:54Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:28:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:28:54Z" name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg.18c00842a206cd55 namespace: kserve-ci-e2e-test resourceVersion: "19403" uid: c88adb9a-38de-4c4b-b44b-e6e86ac773fb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 37620da4-15da-4d0d-ad8f-e404d3b8e339 kind: Event lastTimestamp: "2026-07-07T14:28:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:28:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:28:54Z" name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg.18c00842a56e8f92 namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: f4691ac8-ca9b-4647-a646-ccedffe5c813 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 37620da4-15da-4d0d-ad8f-e404d3b8e339 kind: Event lastTimestamp: "2026-07-07T14:28:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:28:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:28:54Z" name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg.18c00842a5dd0851 namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: c04050c6-5c78-4844-bbb8-6f8f94c8f98f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 37620da4-15da-4d0d-ad8f-e404d3b8e339 kind: Event lastTimestamp: "2026-07-07T14:28:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:28:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:28:54Z" name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg.18c00842a5e5d2b7 namespace: kserve-ci-e2e-test resourceVersion: "19410" uid: ed5d325a-cbda-4548-ab79-2816a4294f8d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 37620da4-15da-4d0d-ad8f-e404d3b8e339 kind: Event lastTimestamp: "2026-07-07T14:28:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:28:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:28:54Z" name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg.18c00842a852951f namespace: kserve-ci-e2e-test resourceVersion: "19412" uid: 93f38dd2-617c-4148-b67b-32e46feeedc3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 37620da4-15da-4d0d-ad8f-e404d3b8e339 kind: Event lastTimestamp: "2026-07-07T14:28:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:28:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:28:54Z" name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg.18c00842a8beb3f7 namespace: kserve-ci-e2e-test resourceVersion: "19413" uid: 9f9ab213-1f87-46be-806d-ae094f17a27a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T14:28:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 37620da4-15da-4d0d-ad8f-e404d3b8e339 kind: Event lastTimestamp: "2026-07-07T14:30:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:28:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:30:40Z" name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg.18c008431a03a30a namespace: kserve-ci-e2e-test resourceVersion: "20266" uid: c4e300e2-eacb-46aa-bc70-05a233c1cbb1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 37620da4-15da-4d0d-ad8f-e404d3b8e339 kind: Event lastTimestamp: "2026-07-07T14:30:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:30:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:30:36Z" name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg.18c0085a48f17d4f namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: a56c1a98-3b4a-4768-b264-182acc69a283 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 37620da4-15da-4d0d-ad8f-e404d3b8e339 kind: Event lastTimestamp: "2026-07-07T14:30:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:30:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:30:36Z" name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg.18c0085a48f2067e namespace: kserve-ci-e2e-test resourceVersion: "20223" uid: 8fcef1e3-75a9-47a5-966b-6b737669649d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 37620da4-15da-4d0d-ad8f-e404d3b8e339 kind: Event lastTimestamp: "2026-07-07T14:30:37Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:30:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:30:37Z" name: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg.18c0085a9e4e593a namespace: kserve-ci-e2e-test resourceVersion: "20252" uid: 759d41c7-c975-4932-ac43-22ad5f9c3b7d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-07975f-predictor-8574dbdbd7 namespace: kserve-ci-e2e-test resourceVersion: "19334" uid: ac77c4f4-5d39-45ca-ba80-2d9543b983e0 kind: Event lastTimestamp: "2026-07-07T14:28:50Z" message: 'Created pod: isvc-primary-07975f-predictor-8574dbdbd7-fvkgg' metadata: creationTimestamp: "2026-07-07T14:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:28:50Z" name: isvc-primary-07975f-predictor-8574dbdbd7.18c008418e3b5531 namespace: kserve-ci-e2e-test resourceVersion: "19347" uid: 3458081c-7ce5-4fc3-9893-3b5e67078866 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-07975f-predictor namespace: kserve-ci-e2e-test resourceVersion: "19332" uid: d00109ac-bb83-4367-b6f3-193318eddfd5 kind: Event lastTimestamp: "2026-07-07T14:28:50Z" message: Scaled up replica set isvc-primary-07975f-predictor-8574dbdbd7 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:28:50Z" name: isvc-primary-07975f-predictor.18c0084189947905 namespace: kserve-ci-e2e-test resourceVersion: "19335" uid: f5b83921-4992-4191-8637-1e5befbd43f2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:29:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-07975f-predictor namespace: kserve-ci-e2e-test resourceVersion: "19333" uid: ac01efb9-b5b0-4e52-939d-c973952851b8 kind: Event lastTimestamp: "2026-07-07T14:29:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:29:05Z" name: isvc-primary-07975f-predictor.18c00845090efe55 namespace: kserve-ci-e2e-test resourceVersion: "19486" uid: 2c35e8bb-6fc4-4342-91d5-978df6fa83fc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:29:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-07975f-predictor namespace: kserve-ci-e2e-test resourceVersion: "19333" uid: ac01efb9-b5b0-4e52-939d-c973952851b8 kind: Event lastTimestamp: "2026-07-07T14:29:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:29:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:29:05Z" name: isvc-primary-07975f-predictor.18c00845097c0d65 namespace: kserve-ci-e2e-test resourceVersion: "19488" uid: ea644481-83f3-4fd7-b912-f4e9ce98afbe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:29:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-07975f-predictor namespace: kserve-ci-e2e-test resourceVersion: "19487" uid: ac01efb9-b5b0-4e52-939d-c973952851b8 kind: Event lastTimestamp: "2026-07-07T14:30:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:29:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:30:35Z" name: isvc-primary-07975f-predictor.18c00848884686aa namespace: kserve-ci-e2e-test resourceVersion: "20192" uid: 0bf09fb2-7457-4205-b227-c8c9fe5771ed reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:29:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-07975f-predictor namespace: kserve-ci-e2e-test resourceVersion: "19487" uid: ac01efb9-b5b0-4e52-939d-c973952851b8 kind: Event lastTimestamp: "2026-07-07T14:30:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:29:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:30:35Z" name: isvc-primary-07975f-predictor.18c0084888bf0de5 namespace: kserve-ci-e2e-test resourceVersion: "20193" uid: 789ae67c-ae45-415d-984e-80536cb0e805 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T14:28:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-07975f namespace: kserve-ci-e2e-test resourceVersion: "19324" uid: 8dedd921-cd3d-4707-8490-3e33ba61e997 kind: Event lastTimestamp: "2026-07-07T14:30:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:30:10Z" name: isvc-primary-07975f.18c00841899f9083 namespace: kserve-ci-e2e-test resourceVersion: "19956" uid: 4ba78ff0-9d94-4955-b172-fbed5a189724 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-07975f namespace: kserve-ci-e2e-test resourceVersion: "19324" uid: 8dedd921-cd3d-4707-8490-3e33ba61e997 kind: Event lastTimestamp: "2026-07-07T14:28:50Z" message: 'Failed to update status for InferenceService "isvc-primary-07975f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-07975f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:28:50Z" name: isvc-primary-07975f.18c008418a237558 namespace: kserve-ci-e2e-test resourceVersion: "19338" uid: b8314697-bb16-44b4-b3d3-ea3a3cd3f712 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-07975f namespace: kserve-ci-e2e-test resourceVersion: "19324" uid: 8dedd921-cd3d-4707-8490-3e33ba61e997 kind: Event lastTimestamp: "2026-07-07T14:28:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-07975f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:28:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:28:50Z" name: isvc-primary-07975f.18c008418a23b9d1 namespace: kserve-ci-e2e-test resourceVersion: "19339" uid: a650271e-b446-4b21-b838-d2b4c406137f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-07975f namespace: kserve-ci-e2e-test resourceVersion: "19953" uid: 8dedd921-cd3d-4707-8490-3e33ba61e997 kind: Event lastTimestamp: "2026-07-07T14:30:10Z" message: InferenceService [isvc-primary-07975f] is Ready metadata: creationTimestamp: "2026-07-07T14:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:30:10Z" name: isvc-primary-07975f.18c00854345e1246 namespace: kserve-ci-e2e-test resourceVersion: "19954" uid: e49e069d-dfb3-49e4-9597-e444118b017b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:18:33.694714Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14066" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst to ip-10-0-140-100.ec2.internal metadata: creationTimestamp: "2026-07-07T14:18:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:18:33Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007b20a28a165 namespace: kserve-ci-e2e-test resourceVersion: "14070" uid: b5df40a3-4006-44a5-846e-991d48134627 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84448d6d9-ddnp2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:18:34Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14075" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:18:34Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:18:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:18:34Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007b2235e1ef7 namespace: kserve-ci-e2e-test resourceVersion: "14083" uid: b97a13bb-1596-485a-9e05-31bb6befff4a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:18:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:18:34Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:18:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:18:34Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007b22501a4a1 namespace: kserve-ci-e2e-test resourceVersion: "14086" uid: 1badc3a1-5418-47a6-b812-6efe349599f3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:18:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:18:34Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:18:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:18:34Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007b2280edf15 namespace: kserve-ci-e2e-test resourceVersion: "14087" uid: f1afdedc-d3ff-4480-b1fa-9a9672e7c847 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:18:34Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:18:34Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:18:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:18:34Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007b2286bb1fa namespace: kserve-ci-e2e-test resourceVersion: "14088" uid: 38db79f5-be01-40f8-a4cb-b7167471a1af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:18:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:18:38Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" metadata: creationTimestamp: "2026-07-07T14:18:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:18:38Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007b314685db6 namespace: kserve-ci-e2e-test resourceVersion: "14121" uid: 960b4957-c88a-4473-9691-1243fbc64f3a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:18:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:18:53Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" in 15.051s (15.051s including waiting). Image size: 1555845045 bytes.' metadata: creationTimestamp: "2026-07-07T14:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:18:53Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007b6958519e2 namespace: kserve-ci-e2e-test resourceVersion: "14229" uid: 9ac34b65-4183-40f2-ad55-3aeb1b41bc3c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:18:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:18:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:18:53Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007b6987daa63 namespace: kserve-ci-e2e-test resourceVersion: "14230" uid: 46938f32-b396-4d12-97c9-353bf5ed3afa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:18:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:18:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:18:53Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007b698ef1f1c namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 6b6b2441-2c89-4890-b3c5-422b8238a46b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:18:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:18:53Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-07T14:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:18:53Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007b698fb7937 namespace: kserve-ci-e2e-test resourceVersion: "14232" uid: b29b0fc5-cc91-4c8b-a8cb-ce47fb4fdc17 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:18:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:18:56Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.986s (2.986s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-07T14:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:18:56Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007b74b047f7b namespace: kserve-ci-e2e-test resourceVersion: "14256" uid: 5a150566-b5ec-4638-91f1-dda86fdecbf1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:18:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:18:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:18:56Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007b7509ed25b namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 2af7d0c5-b403-4f0c-afdc-1c987d8b54cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:18:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:18:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:18:56Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007b7525046e7 namespace: kserve-ci-e2e-test resourceVersion: "14258" uid: f9d11a20-0c03-4296-9c8d-929091331da6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:18:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:18:56Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:0b6de7012f6344e0be5c95ac8deb96d56b96edd624ee25e9ad8ecbad40286f86" metadata: creationTimestamp: "2026-07-07T14:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:18:56Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007b7525eba75 namespace: kserve-ci-e2e-test resourceVersion: "14259" uid: 3e4c64a7-6b38-4754-a036-d21983a16072 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:18:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:18:59Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:0b6de7012f6344e0be5c95ac8deb96d56b96edd624ee25e9ad8ecbad40286f86" in 3.125s (3.125s including waiting). Image size: 239997051 bytes.' metadata: creationTimestamp: "2026-07-07T14:18:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:18:59Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007b80ca65905 namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 78c38050-533b-4da4-ac4f-6e8b5a9459de reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:18:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:18:59Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-07T14:18:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:18:59Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007b811a522dc namespace: kserve-ci-e2e-test resourceVersion: "14280" uid: 30ad3a6b-c471-4bfc-b280-29cbe343147d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:18:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:18:59Z" message: Started container agent metadata: creationTimestamp: "2026-07-07T14:18:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:18:59Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007b812d1befc namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 5278f17e-8538-4f19-bd5e-875d0ee55177 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-07T14:19:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:20:43Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:19:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:43Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007b83a9a41e4 namespace: kserve-ci-e2e-test resourceVersion: "15176" uid: 307fbe9f-8c66-4f2d-851d-e7243678e793 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T14:19:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:20:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T14:19:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:33Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007b8766b617b namespace: kserve-ci-e2e-test resourceVersion: "15085" uid: e33414de-819b-43f3-a710-589372c91042 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:20:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:20:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:28Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007ccd9f8ddce namespace: kserve-ci-e2e-test resourceVersion: "14983" uid: a8a87ba0-9d45-4a17-a230-2d11f8a85051 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:20:28Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-07T14:20:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:28Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007ccd9f90d75 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: e724c8c8-b84b-4217-a4ed-6a8f80675dc4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:20:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:20:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:28Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007ccd9f974c8 namespace: kserve-ci-e2e-test resourceVersion: "14988" uid: 6ddf4cf5-0635-4ac7-b02d-33d647bf99f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:20:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: c234c601-fef9-45f3-9668-ab81c7494147 kind: Event lastTimestamp: "2026-07-07T14:20:41Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:20:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:41Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst.18c007cd6acd6c55 namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: 051f5f64-ef57-405d-904e-d33c5b02c0b4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:18:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54 namespace: kserve-ci-e2e-test resourceVersion: "14047" uid: 718d1a81-8b44-4b36-ba84-6ff3b1d4039c kind: Event lastTimestamp: "2026-07-07T14:18:33Z" message: 'Created pod: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54-d6gst' metadata: creationTimestamp: "2026-07-07T14:18:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:18:33Z" name: isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54.18c007b209a05f43 namespace: kserve-ci-e2e-test resourceVersion: "14067" uid: c567956f-86fa-40c7-852a-3b1690a714cb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:18:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-61b8a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14046" uid: a587b7ca-18ff-440c-90b5-259455cc155f kind: Event lastTimestamp: "2026-07-07T14:18:33Z" message: Scaled up replica set isvc-raw-sklearn-batcher-61b8a-predictor-745cfc7d54 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:18:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:18:33Z" name: isvc-raw-sklearn-batcher-61b8a-predictor.18c007b1fb31e1a9 namespace: kserve-ci-e2e-test resourceVersion: "14048" uid: 65297d56-dfd7-4db4-bbb3-04093c7988ea reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:18:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-61b8a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14056" uid: acbf7f22-959d-4dd0-959c-1b49e20a86fa kind: Event lastTimestamp: "2026-07-07T14:19:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:18:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:19:33Z" name: isvc-raw-sklearn-batcher-61b8a-predictor.18c007b583075064 namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: 20795399-5c72-4c7d-bf2d-2145752292f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:18:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-61b8a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14056" uid: acbf7f22-959d-4dd0-959c-1b49e20a86fa kind: Event lastTimestamp: "2026-07-07T14:19:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:18:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:19:33Z" name: isvc-raw-sklearn-batcher-61b8a-predictor.18c007b583b75d91 namespace: kserve-ci-e2e-test resourceVersion: "14540" uid: 75e5c02e-fef4-4ea5-ad39-13d5ccf189dd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:19:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-61b8a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14197" uid: acbf7f22-959d-4dd0-959c-1b49e20a86fa kind: Event lastTimestamp: "2026-07-07T14:20:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:19:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:20:18Z" name: isvc-raw-sklearn-batcher-61b8a-predictor.18c007bc826247bf namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: ac046f61-8459-4f40-a5f6-3e3fc5d5e3a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:19:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-61b8a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14197" uid: acbf7f22-959d-4dd0-959c-1b49e20a86fa kind: Event lastTimestamp: "2026-07-07T14:20:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:19:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:20:18Z" name: isvc-raw-sklearn-batcher-61b8a-predictor.18c007bc82e00a6c namespace: kserve-ci-e2e-test resourceVersion: "14871" uid: 5f22e5d5-bf63-4f0a-8e30-a58c24083c84 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T14:18:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-61b8a namespace: kserve-ci-e2e-test resourceVersion: "14037" uid: 5e50bcee-9d95-47d8-b655-9ede7638fc01 kind: Event lastTimestamp: "2026-07-07T14:20:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:18:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:20:13Z" name: isvc-raw-sklearn-batcher-61b8a.18c007b203001684 namespace: kserve-ci-e2e-test resourceVersion: "14832" uid: 8ddb28ee-d58f-4abc-a248-2e9a825def88 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:18:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-61b8a namespace: kserve-ci-e2e-test resourceVersion: "14037" uid: 5e50bcee-9d95-47d8-b655-9ede7638fc01 kind: Event lastTimestamp: "2026-07-07T14:18:33Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-61b8a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-61b8a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:18:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:18:33Z" name: isvc-raw-sklearn-batcher-61b8a.18c007b203fdcf43 namespace: kserve-ci-e2e-test resourceVersion: "14078" uid: 8f02524c-1c46-4ae2-88f8-83232c364655 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:18:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-61b8a namespace: kserve-ci-e2e-test resourceVersion: "14037" uid: 5e50bcee-9d95-47d8-b655-9ede7638fc01 kind: Event lastTimestamp: "2026-07-07T14:18:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-61b8a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:18:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:18:33Z" name: isvc-raw-sklearn-batcher-61b8a.18c007b203fe3222 namespace: kserve-ci-e2e-test resourceVersion: "14079" uid: f9b72448-7a2d-49c7-b4d3-addd281e1ea7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-61b8a namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: 5e50bcee-9d95-47d8-b655-9ede7638fc01 kind: Event lastTimestamp: "2026-07-07T14:20:13Z" message: InferenceService [isvc-raw-sklearn-batcher-61b8a] is Ready metadata: creationTimestamp: "2026-07-07T14:20:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:20:13Z" name: isvc-raw-sklearn-batcher-61b8a.18c007c94bb56f36 namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: b923cc15-665b-4155-b3c9-ec77b5bf37b4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:30:20.329040Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-07975f-predictor-7b8f47b784-gz77k namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: 8b6caba9-a942-443b-8dbe-0b462a60d80b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-07975f-predictor-7b8f47b784-gz77k to ip-10-0-140-100.ec2.internal metadata: creationTimestamp: "2026-07-07T14:30:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:30:20Z" name: isvc-secondary-07975f-predictor-7b8f47b784-gz77k.18c0085690d8aad4 namespace: kserve-ci-e2e-test resourceVersion: "20055" uid: 77e5ed4c-f783-4afe-9605-977ba9d6d378 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84448d6d9-ddnp2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-07975f-predictor-7b8f47b784-gz77k namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: 8b6caba9-a942-443b-8dbe-0b462a60d80b kind: Event lastTimestamp: "2026-07-07T14:30:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-secondary-07975f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:30:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:30:20Z" name: isvc-secondary-07975f-predictor-7b8f47b784-gz77k.18c008569d2a24fb namespace: kserve-ci-e2e-test resourceVersion: "20066" uid: d3e2e9cf-acf6-4c5f-b19a-3ef7a3d8442d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:21Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-07975f-predictor-7b8f47b784-gz77k namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 8b6caba9-a942-443b-8dbe-0b462a60d80b kind: Event lastTimestamp: "2026-07-07T14:30:21Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:30:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:30:21Z" name: isvc-secondary-07975f-predictor-7b8f47b784-gz77k.18c00856cd9e19f8 namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: 8a1ca864-82d0-43db-97b7-7840a97be8fb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:30:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-07975f-predictor-7b8f47b784-gz77k namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: 8b6caba9-a942-443b-8dbe-0b462a60d80b kind: Event lastTimestamp: "2026-07-07T14:30:25Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:30:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:30:25Z" name: isvc-secondary-07975f-predictor-7b8f47b784-gz77k.18c00856cf001725 namespace: kserve-ci-e2e-test resourceVersion: "20109" uid: 3f1a2d82-e3c0-4756-bf80-f4bc99d98dbc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:30:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-07975f-predictor-7b8f47b784-gz77k namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: 8b6caba9-a942-443b-8dbe-0b462a60d80b kind: Event lastTimestamp: "2026-07-07T14:30:25Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:30:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:30:25Z" name: isvc-secondary-07975f-predictor-7b8f47b784-gz77k.18c00856d18a5fa4 namespace: kserve-ci-e2e-test resourceVersion: "20111" uid: baad3587-0234-4426-be45-c62e941938c2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:30:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-07975f-predictor-7b8f47b784-gz77k namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: 8b6caba9-a942-443b-8dbe-0b462a60d80b kind: Event lastTimestamp: "2026-07-07T14:30:25Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:30:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:30:25Z" name: isvc-secondary-07975f-predictor-7b8f47b784-gz77k.18c00856d1f79962 namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: 97feca9c-90cc-4b39-9f41-087b1fb7816d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:31Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-07975f-predictor-7b8f47b784-gz77k namespace: kserve-ci-e2e-test resourceVersion: "20054" uid: 8b6caba9-a942-443b-8dbe-0b462a60d80b kind: Event lastTimestamp: "2026-07-07T14:30:31Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-07975f-predictor-7b8f47b784-gz77k_kserve-ci-e2e-test(8b6caba9-a942-443b-8dbe-0b462a60d80b) metadata: creationTimestamp: "2026-07-07T14:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:30:31Z" name: isvc-secondary-07975f-predictor-7b8f47b784-gz77k.18c00859142768a9 namespace: kserve-ci-e2e-test resourceVersion: "20159" uid: e4562510-8bd9-4a4d-acda-1b284739992c reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-07975f-predictor-7b8f47b784 namespace: kserve-ci-e2e-test resourceVersion: "20041" uid: 864a6a41-eb56-4e2a-8c26-637174b9d556 kind: Event lastTimestamp: "2026-07-07T14:30:20Z" message: 'Created pod: isvc-secondary-07975f-predictor-7b8f47b784-gz77k' metadata: creationTimestamp: "2026-07-07T14:30:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:30:20Z" name: isvc-secondary-07975f-predictor-7b8f47b784.18c00856900a82c0 namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: 9a757a86-1826-4b4c-9f80-9c4d8c731f2e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-07975f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20039" uid: 0100583f-02b4-44a1-a3a1-f6d99282603b kind: Event lastTimestamp: "2026-07-07T14:30:20Z" message: Scaled up replica set isvc-secondary-07975f-predictor-7b8f47b784 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:30:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:30:20Z" name: isvc-secondary-07975f-predictor.18c008568b287a3e namespace: kserve-ci-e2e-test resourceVersion: "20043" uid: 53f2f923-2096-42b8-bb47-884deecfa8fe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-07975f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: ec147dd2-f995-48e2-bba7-709fc3e6eb41 kind: Event lastTimestamp: "2026-07-07T14:30:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:30:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:30:35Z" name: isvc-secondary-07975f-predictor.18c0085a097d6205 namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: bcc251f0-d4fe-40d4-a5a5-63443832da3a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-07975f-predictor namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: ec147dd2-f995-48e2-bba7-709fc3e6eb41 kind: Event lastTimestamp: "2026-07-07T14:30:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:30:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:30:35Z" name: isvc-secondary-07975f-predictor.18c0085a09eea156 namespace: kserve-ci-e2e-test resourceVersion: "20196" uid: 9fdcef90-7782-4643-a4b9-bfcc9c8647eb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-07-07T14:30:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-07975f namespace: kserve-ci-e2e-test resourceVersion: "20032" uid: e0725239-742d-475a-a2d2-936a1cdeff39 kind: Event lastTimestamp: "2026-07-07T14:30:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:30:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:30:31Z" name: isvc-secondary-07975f.18c008568af76b5a namespace: kserve-ci-e2e-test resourceVersion: "20166" uid: cf77bff8-257d-431a-8ccd-a3d789c7524b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-07975f namespace: kserve-ci-e2e-test resourceVersion: "20032" uid: e0725239-742d-475a-a2d2-936a1cdeff39 kind: Event lastTimestamp: "2026-07-07T14:30:20Z" message: 'Failed to update status for InferenceService "isvc-secondary-07975f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-07975f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:30:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:30:20Z" name: isvc-secondary-07975f.18c008568b4e0abf namespace: kserve-ci-e2e-test resourceVersion: "20045" uid: d2558524-ecd4-45e9-ad01-5d263bdfc2b6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-07975f namespace: kserve-ci-e2e-test resourceVersion: "20032" uid: e0725239-742d-475a-a2d2-936a1cdeff39 kind: Event lastTimestamp: "2026-07-07T14:30:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-07975f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:30:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:30:20Z" name: isvc-secondary-07975f.18c008568b4e6fe9 namespace: kserve-ci-e2e-test resourceVersion: "20046" uid: a716c98f-0a34-406d-ba66-06876021dfc2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:20:28.900587Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: a2003c14-59bd-4497-9988-f52f04cfff1b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj to ip-10-0-140-100.ec2.internal metadata: creationTimestamp: "2026-07-07T14:20:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:20:28Z" name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj.18c007ccdcf6aacb namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: e8329cf4-9bf5-4728-a253-760fda08b22b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84448d6d9-ddnp2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:29Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj namespace: kserve-ci-e2e-test resourceVersion: "15005" uid: a2003c14-59bd-4497-9988-f52f04cfff1b kind: Event lastTimestamp: "2026-07-07T14:20:29Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:20:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:20:29Z" name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj.18c007ccf69c7607 namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: 4c05bcc8-8fbd-4efb-a358-60f7c0edc7f8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: a2003c14-59bd-4497-9988-f52f04cfff1b kind: Event lastTimestamp: "2026-07-07T14:20:29Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:20:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:29Z" name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj.18c007ccf8b216c8 namespace: kserve-ci-e2e-test resourceVersion: "15041" uid: 568c560b-7b23-4dea-b010-fe94f53e4d66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: a2003c14-59bd-4497-9988-f52f04cfff1b kind: Event lastTimestamp: "2026-07-07T14:20:29Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:20:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:29Z" name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj.18c007ccfc8f783b namespace: kserve-ci-e2e-test resourceVersion: "15043" uid: d51d89d0-2cd9-45be-86b2-1d984d1bd31a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: a2003c14-59bd-4497-9988-f52f04cfff1b kind: Event lastTimestamp: "2026-07-07T14:20:29Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:20:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:29Z" name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj.18c007ccfcff844c namespace: kserve-ci-e2e-test resourceVersion: "15045" uid: 34f9fd3d-a64b-41b8-b844-978a33758f64 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: a2003c14-59bd-4497-9988-f52f04cfff1b kind: Event lastTimestamp: "2026-07-07T14:20:33Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:20:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:33Z" name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj.18c007cdf82dc16e namespace: kserve-ci-e2e-test resourceVersion: "15089" uid: b5cf4055-ebdc-4f35-808e-58876e898086 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: a2003c14-59bd-4497-9988-f52f04cfff1b kind: Event lastTimestamp: "2026-07-07T14:20:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:20:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:33Z" name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj.18c007cdfc0af3d5 namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: c8255f0a-1479-4c85-97cf-0b2da43994be reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: a2003c14-59bd-4497-9988-f52f04cfff1b kind: Event lastTimestamp: "2026-07-07T14:20:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:20:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:33Z" name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj.18c007cdfca031d7 namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: 2495cf1b-1c3a-4e2d-9f5f-66c2e267bdff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: a2003c14-59bd-4497-9988-f52f04cfff1b kind: Event lastTimestamp: "2026-07-07T14:20:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:20:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:33Z" name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj.18c007cdfcaa6317 namespace: kserve-ci-e2e-test resourceVersion: "15096" uid: d0df8f98-a395-4b3f-87f6-90e00dfedb7e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: a2003c14-59bd-4497-9988-f52f04cfff1b kind: Event lastTimestamp: "2026-07-07T14:20:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:20:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:33Z" name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj.18c007cdff5f49ee namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 5fc379c6-4c7f-4033-bb23-48c68f45cf04 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: a2003c14-59bd-4497-9988-f52f04cfff1b kind: Event lastTimestamp: "2026-07-07T14:20:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:20:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:33Z" name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj.18c007cdffc9ca8d namespace: kserve-ci-e2e-test resourceVersion: "15101" uid: 83520b3a-5542-4fff-9378-181c6480340f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T14:20:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: a2003c14-59bd-4497-9988-f52f04cfff1b kind: Event lastTimestamp: "2026-07-07T14:22:21Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:20:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:21Z" name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj.18c007ce702fe136 namespace: kserve-ci-e2e-test resourceVersion: "16105" uid: 4bb7e351-1809-4774-a31d-162c76160c27 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: a2003c14-59bd-4497-9988-f52f04cfff1b kind: Event lastTimestamp: "2026-07-07T14:22:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:22:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:19Z" name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj.18c007e68e319163 namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: 60b25c15-2c40-4283-aace-8aac9f699d4d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: a2003c14-59bd-4497-9988-f52f04cfff1b kind: Event lastTimestamp: "2026-07-07T14:22:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:22:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:19Z" name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj.18c007e68e32106a namespace: kserve-ci-e2e-test resourceVersion: "16011" uid: f15e500c-00fe-4cb2-b51f-c163de3d67b3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: a2003c14-59bd-4497-9988-f52f04cfff1b kind: Event lastTimestamp: "2026-07-07T14:22:21Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:21Z" name: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj.18c007e71e897ec2 namespace: kserve-ci-e2e-test resourceVersion: "16104" uid: 33b0769b-ceb5-45ba-83d8-9f78067a27eb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-884de-predictor-7678456578 namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: a0c9e7b9-4ca0-404b-a3d7-e95d931e97ed kind: Event lastTimestamp: "2026-07-07T14:20:28Z" message: 'Created pod: isvc-sklearn-graph-raw-884de-predictor-7678456578-zh5nj' metadata: creationTimestamp: "2026-07-07T14:20:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:20:28Z" name: isvc-sklearn-graph-raw-884de-predictor-7678456578.18c007ccdbf86dd3 namespace: kserve-ci-e2e-test resourceVersion: "14990" uid: e9ca467f-3242-4d95-bbe5-2c4c38681885 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-884de-predictor namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 0f6b8732-78cd-45ad-ba39-218839ffc294 kind: Event lastTimestamp: "2026-07-07T14:20:28Z" message: Scaled up replica set isvc-sklearn-graph-raw-884de-predictor-7678456578 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:20:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:20:28Z" name: isvc-sklearn-graph-raw-884de-predictor.18c007ccd3a3411a namespace: kserve-ci-e2e-test resourceVersion: "14961" uid: 1e695d33-341f-4af4-aba0-f2ccc14970b2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:20:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-884de-predictor namespace: kserve-ci-e2e-test resourceVersion: "14957" uid: 82fd8247-4016-4282-a4ac-2d4ea9f439c0 kind: Event lastTimestamp: "2026-07-07T14:20:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:20:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:20:58Z" name: isvc-sklearn-graph-raw-884de-predictor.18c007d05238b618 namespace: kserve-ci-e2e-test resourceVersion: "15303" uid: 6a48aa82-417f-4c15-a007-4cb095c45cb8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:20:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-884de-predictor namespace: kserve-ci-e2e-test resourceVersion: "14957" uid: 82fd8247-4016-4282-a4ac-2d4ea9f439c0 kind: Event lastTimestamp: "2026-07-07T14:20:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:20:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:20:58Z" name: isvc-sklearn-graph-raw-884de-predictor.18c007d052af9b25 namespace: kserve-ci-e2e-test resourceVersion: "15304" uid: ec58161d-adc8-4e11-8660-e13972830342 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T14:21:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-884de-predictor namespace: kserve-ci-e2e-test resourceVersion: "15181" uid: 82fd8247-4016-4282-a4ac-2d4ea9f439c0 kind: Event lastTimestamp: "2026-07-07T14:22:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:21:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:22:13Z" name: isvc-sklearn-graph-raw-884de-predictor.18c007d75291c3d7 namespace: kserve-ci-e2e-test resourceVersion: "15902" uid: 32c0eb40-4fb2-487c-ac93-7c8ed5b210bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T14:21:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-884de-predictor namespace: kserve-ci-e2e-test resourceVersion: "15181" uid: 82fd8247-4016-4282-a4ac-2d4ea9f439c0 kind: Event lastTimestamp: "2026-07-07T14:22:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:21:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:22:13Z" name: isvc-sklearn-graph-raw-884de-predictor.18c007d7532712e1 namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: fc1bf025-45fc-41c4-8296-4bd500b52c60 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T14:20:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-884de namespace: kserve-ci-e2e-test resourceVersion: "14946" uid: f3d948c4-1439-4254-b5d4-4c7d20e522e9 kind: Event lastTimestamp: "2026-07-07T14:21:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:20:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:21:51Z" name: isvc-sklearn-graph-raw-884de.18c007ccd2d2e9e0 namespace: kserve-ci-e2e-test resourceVersion: "15703" uid: 72231037-5a05-4cd3-9dc4-f2c023831c92 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-884de namespace: kserve-ci-e2e-test resourceVersion: "14946" uid: f3d948c4-1439-4254-b5d4-4c7d20e522e9 kind: Event lastTimestamp: "2026-07-07T14:20:28Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-884de": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-884de": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:20:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:20:28Z" name: isvc-sklearn-graph-raw-884de.18c007ccd3d267fe namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: 0b2aeb77-bfa0-4dd1-b2cf-60d5664ce837 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:21:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-884de namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: f3d948c4-1439-4254-b5d4-4c7d20e522e9 kind: Event lastTimestamp: "2026-07-07T14:21:51Z" message: InferenceService [isvc-sklearn-graph-raw-884de] is Ready metadata: creationTimestamp: "2026-07-07T14:21:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:21:51Z" name: isvc-sklearn-graph-raw-884de.18c007e02a8123ae namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: 24480bc2-1167-4ff0-adf8-0fd9f2f229e9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:22:19.249024Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9 namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: 7fe340e8-5229-4970-b432-41feb1f12ed2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9 to ip-10-0-140-100.ec2.internal metadata: creationTimestamp: "2026-07-07T14:22:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:22:19Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9.18c007e68e3e3353 namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: c4c2415a-f19d-4a98-867a-21dac5f2067a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84448d6d9-ddnp2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:19Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9 namespace: kserve-ci-e2e-test resourceVersion: "16019" uid: 7fe340e8-5229-4970-b432-41feb1f12ed2 kind: Event lastTimestamp: "2026-07-07T14:22:19Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:22:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:22:19Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9.18c007e6a7d7c717 namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: fa58b3ef-f4ce-40d5-b7b6-05e97b105cff reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 7fe340e8-5229-4970-b432-41feb1f12ed2 kind: Event lastTimestamp: "2026-07-07T14:22:19Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:22:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:19Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9.18c007e6a9692980 namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: b29bea4f-8bcc-4c74-970e-fdd1e7624414 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 7fe340e8-5229-4970-b432-41feb1f12ed2 kind: Event lastTimestamp: "2026-07-07T14:22:19Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:22:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:19Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9.18c007e6abaed861 namespace: kserve-ci-e2e-test resourceVersion: "16072" uid: aa5b5822-04e3-4a61-b563-868dce67d6e3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 7fe340e8-5229-4970-b432-41feb1f12ed2 kind: Event lastTimestamp: "2026-07-07T14:22:19Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:22:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:19Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9.18c007e6ac0e9b64 namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: 2d479a01-30d1-4401-81ea-96a3b2e6c0a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 7fe340e8-5229-4970-b432-41feb1f12ed2 kind: Event lastTimestamp: "2026-07-07T14:22:24Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:24Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9.18c007e7b1a2c708 namespace: kserve-ci-e2e-test resourceVersion: "16129" uid: 8491a33d-4ef8-4f5c-afc1-b160c2632e84 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 7fe340e8-5229-4970-b432-41feb1f12ed2 kind: Event lastTimestamp: "2026-07-07T14:22:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:24Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9.18c007e7b486ac1b namespace: kserve-ci-e2e-test resourceVersion: "16134" uid: 4ccf7b5e-898c-4a43-8a8b-0a2d1d91a1ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 7fe340e8-5229-4970-b432-41feb1f12ed2 kind: Event lastTimestamp: "2026-07-07T14:22:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:24Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9.18c007e7b4fe9c13 namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 23c75252-70bf-4d6b-b4f5-385334d1c264 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 7fe340e8-5229-4970-b432-41feb1f12ed2 kind: Event lastTimestamp: "2026-07-07T14:22:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:24Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9.18c007e7b5086655 namespace: kserve-ci-e2e-test resourceVersion: "16138" uid: 15b5b37f-75f6-46d0-b349-29e0a538a755 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 7fe340e8-5229-4970-b432-41feb1f12ed2 kind: Event lastTimestamp: "2026-07-07T14:22:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:24Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9.18c007e7b7e06792 namespace: kserve-ci-e2e-test resourceVersion: "16144" uid: 81caf43a-22e0-4b04-ae9f-f96dcecc8196 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 7fe340e8-5229-4970-b432-41feb1f12ed2 kind: Event lastTimestamp: "2026-07-07T14:22:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:24Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9.18c007e7b8519135 namespace: kserve-ci-e2e-test resourceVersion: "16147" uid: 83a3f633-4314-4575-992e-0ebbc85e2902 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T14:22:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 7fe340e8-5229-4970-b432-41feb1f12ed2 kind: Event lastTimestamp: "2026-07-07T14:24:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:12Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9.18c007e829e6ccc7 namespace: kserve-ci-e2e-test resourceVersion: "17110" uid: 47c2f2a2-b612-4eda-91b4-e4b6fe3fd4b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 7fe340e8-5229-4970-b432-41feb1f12ed2 kind: Event lastTimestamp: "2026-07-07T14:24:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:09Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9.18c0080038fcd866 namespace: kserve-ci-e2e-test resourceVersion: "17062" uid: a4769d94-59ca-40a0-8447-606e61b7d920 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 7fe340e8-5229-4970-b432-41feb1f12ed2 kind: Event lastTimestamp: "2026-07-07T14:24:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:09Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9.18c0080038fe003f namespace: kserve-ci-e2e-test resourceVersion: "17066" uid: b7e6e003-572a-4eb2-aaaf-5fb649327ba0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 7fe340e8-5229-4970-b432-41feb1f12ed2 kind: Event lastTimestamp: "2026-07-07T14:24:12Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:24:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:12Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9.18c00800d84ee7ca namespace: kserve-ci-e2e-test resourceVersion: "17108" uid: e1710681-2dd7-4908-a7f5-b2de54bdd35d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98 namespace: kserve-ci-e2e-test resourceVersion: "15977" uid: 80302e8e-6ae5-40e4-9aff-9b28ac6b9b37 kind: Event lastTimestamp: "2026-07-07T14:22:19Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98-c6lp9' metadata: creationTimestamp: "2026-07-07T14:22:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:22:19Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98.18c007e68e4afa0f namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: cae9d6f4-24a2-4a56-bc9c-3060ad79517d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-bd33a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: be6d6b39-c6ff-452a-b2f6-08846993c8ee kind: Event lastTimestamp: "2026-07-07T14:22:19Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-bd33a-predictor-5f66864c98 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:22:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:22:19Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor.18c007e6888c0c3d namespace: kserve-ci-e2e-test resourceVersion: "15979" uid: 4e0573e6-626e-43d4-9948-a844f8e9d8c7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:22:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-bd33a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: 6ec403b2-7b18-415a-aa4c-4780c349e5ed kind: Event lastTimestamp: "2026-07-07T14:23:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:22:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:04Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor.18c007ea07fdac1a namespace: kserve-ci-e2e-test resourceVersion: "16456" uid: 2e15ab8b-f31a-493a-9775-4f9052b08857 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:22:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-bd33a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: 6ec403b2-7b18-415a-aa4c-4780c349e5ed kind: Event lastTimestamp: "2026-07-07T14:23:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:22:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:04Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor.18c007ea08843d9b namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: fca6b35a-0e5e-4388-82c4-12d85594d9b3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T14:22:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-bd33a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: 6ec403b2-7b18-415a-aa4c-4780c349e5ed kind: Event lastTimestamp: "2026-07-07T14:24:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:22:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:04Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor.18c007ed877469e6 namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 4b640368-f644-451d-990e-9d52c3d5cde5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T14:22:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-bd33a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: 6ec403b2-7b18-415a-aa4c-4780c349e5ed kind: Event lastTimestamp: "2026-07-07T14:24:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:22:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:04Z" name: isvc-sklearn-graph-raw-hpa-bd33a-predictor.18c007ed87eceda6 namespace: kserve-ci-e2e-test resourceVersion: "16949" uid: 9bbd732a-2d3b-4bdc-93ad-9de67e695f55 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-07T14:22:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-bd33a namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: c4fab312-9d77-40e1-90b1-fb3d80ebc1cc kind: Event lastTimestamp: "2026-07-07T14:23:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:22:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:23:42Z" name: isvc-sklearn-graph-raw-hpa-bd33a.18c007e688b714c3 namespace: kserve-ci-e2e-test resourceVersion: "16755" uid: 307d0c71-faa5-43c3-bef4-c99983d8f6a7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-bd33a namespace: kserve-ci-e2e-test resourceVersion: "16753" uid: c4fab312-9d77-40e1-90b1-fb3d80ebc1cc kind: Event lastTimestamp: "2026-07-07T14:23:42Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-bd33a] is Ready metadata: creationTimestamp: "2026-07-07T14:23:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:23:42Z" name: isvc-sklearn-graph-raw-hpa-bd33a.18c007f9e28b0656 namespace: kserve-ci-e2e-test resourceVersion: "16754" uid: 2bf1b697-1a2b-43e2-bf76-9a5de2336850 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:26:04.985251Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8 namespace: kserve-ci-e2e-test resourceVersion: "18059" uid: 9d182cc8-09e3-46d9-9445-1b2f2eb9fd3e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8 to ip-10-0-140-100.ec2.internal metadata: creationTimestamp: "2026-07-07T14:26:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:26:04Z" name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8.18c0081b1d2ba9d9 namespace: kserve-ci-e2e-test resourceVersion: "18072" uid: 7c8fd08c-a0b2-4072-b888-0ca673f39e10 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84448d6d9-ddnp2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8 namespace: kserve-ci-e2e-test resourceVersion: "18086" uid: 9d182cc8-09e3-46d9-9445-1b2f2eb9fd3e kind: Event lastTimestamp: "2026-07-07T14:26:05Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:26:05Z" name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8.18c0081b3680a4a8 namespace: kserve-ci-e2e-test resourceVersion: "18100" uid: 47e234f5-737a-458c-87b0-0f7dd6679d24 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8 namespace: kserve-ci-e2e-test resourceVersion: "18066" uid: 9d182cc8-09e3-46d9-9445-1b2f2eb9fd3e kind: Event lastTimestamp: "2026-07-07T14:26:05Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:05Z" name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8.18c0081b38718c49 namespace: kserve-ci-e2e-test resourceVersion: "18102" uid: 65d2d09c-2f48-43cb-b537-231c42f65994 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8 namespace: kserve-ci-e2e-test resourceVersion: "18066" uid: 9d182cc8-09e3-46d9-9445-1b2f2eb9fd3e kind: Event lastTimestamp: "2026-07-07T14:26:05Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:05Z" name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8.18c0081b3b8f81ef namespace: kserve-ci-e2e-test resourceVersion: "18103" uid: 20ef1178-9d26-4321-b4a3-3d1d9a5cc55a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8 namespace: kserve-ci-e2e-test resourceVersion: "18066" uid: 9d182cc8-09e3-46d9-9445-1b2f2eb9fd3e kind: Event lastTimestamp: "2026-07-07T14:26:05Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:05Z" name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8.18c0081b3bfd7140 namespace: kserve-ci-e2e-test resourceVersion: "18104" uid: 858aa522-5019-4001-81f0-8c9c4f5f3092 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8 namespace: kserve-ci-e2e-test resourceVersion: "18066" uid: 9d182cc8-09e3-46d9-9445-1b2f2eb9fd3e kind: Event lastTimestamp: "2026-07-07T14:26:10Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:10Z" name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8.18c0081c4c11e39e namespace: kserve-ci-e2e-test resourceVersion: "18147" uid: 14f6adb8-d945-4a27-b5b9-9e43009e9e2e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8 namespace: kserve-ci-e2e-test resourceVersion: "18066" uid: 9d182cc8-09e3-46d9-9445-1b2f2eb9fd3e kind: Event lastTimestamp: "2026-07-07T14:26:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:10Z" name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8.18c0081c4f2e8292 namespace: kserve-ci-e2e-test resourceVersion: "18149" uid: 6bb3274b-d802-498a-aa4e-65915801986c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8 namespace: kserve-ci-e2e-test resourceVersion: "18066" uid: 9d182cc8-09e3-46d9-9445-1b2f2eb9fd3e kind: Event lastTimestamp: "2026-07-07T14:26:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:10Z" name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8.18c0081c4f98b8b1 namespace: kserve-ci-e2e-test resourceVersion: "18150" uid: 87392d48-4e84-4af9-8f66-003ac63ada00 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8 namespace: kserve-ci-e2e-test resourceVersion: "18066" uid: 9d182cc8-09e3-46d9-9445-1b2f2eb9fd3e kind: Event lastTimestamp: "2026-07-07T14:26:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:10Z" name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8.18c0081c4fa1b3b1 namespace: kserve-ci-e2e-test resourceVersion: "18152" uid: b3eba3da-a9cc-43aa-ba29-22417416c51e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8 namespace: kserve-ci-e2e-test resourceVersion: "18066" uid: 9d182cc8-09e3-46d9-9445-1b2f2eb9fd3e kind: Event lastTimestamp: "2026-07-07T14:26:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:10Z" name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8.18c0081c52a706bd namespace: kserve-ci-e2e-test resourceVersion: "18155" uid: 9e6bf9f3-ab95-473d-a1e0-18425438c724 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8 namespace: kserve-ci-e2e-test resourceVersion: "18066" uid: 9d182cc8-09e3-46d9-9445-1b2f2eb9fd3e kind: Event lastTimestamp: "2026-07-07T14:26:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:26:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:10Z" name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8.18c0081c5323a2c6 namespace: kserve-ci-e2e-test resourceVersion: "18156" uid: bdd7dd10-9d3b-4221-9ac5-6ce6da29c90d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 18 eventTime: null firstTimestamp: "2026-07-07T14:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8 namespace: kserve-ci-e2e-test resourceVersion: "18066" uid: 9d182cc8-09e3-46d9-9445-1b2f2eb9fd3e kind: Event lastTimestamp: "2026-07-07T14:28:51Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:28:51Z" name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8.18c0081cc3feca13 namespace: kserve-ci-e2e-test resourceVersion: "19375" uid: 2b1523ae-32ee-4fde-9ad3-3dd3906aa10f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8 namespace: kserve-ci-e2e-test resourceVersion: "18066" uid: 9d182cc8-09e3-46d9-9445-1b2f2eb9fd3e kind: Event lastTimestamp: "2026-07-07T14:28:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:28:44Z" name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8.18c008405e533957 namespace: kserve-ci-e2e-test resourceVersion: "19286" uid: 885b91f6-2811-4861-9744-469da5af8714 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8 namespace: kserve-ci-e2e-test resourceVersion: "18066" uid: 9d182cc8-09e3-46d9-9445-1b2f2eb9fd3e kind: Event lastTimestamp: "2026-07-07T14:28:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:28:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:28:45Z" name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8.18c008405e53bf81 namespace: kserve-ci-e2e-test resourceVersion: "19287" uid: 7b5dd4bb-79ec-40b8-b20c-f6094a669746 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:28:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8 namespace: kserve-ci-e2e-test resourceVersion: "18066" uid: 9d182cc8-09e3-46d9-9445-1b2f2eb9fd3e kind: Event lastTimestamp: "2026-07-07T14:28:53Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:28:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:28:53Z" name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8.18c00841168feff7 namespace: kserve-ci-e2e-test resourceVersion: "19388" uid: 4796381a-bb1d-4162-8c67-f1ca7708484d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4 namespace: kserve-ci-e2e-test resourceVersion: "18033" uid: b891a273-0901-474a-9371-bea54796c9bc kind: Event lastTimestamp: "2026-07-07T14:26:04Z" message: 'Created pod: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4-ms4z8' metadata: creationTimestamp: "2026-07-07T14:26:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:26:04Z" name: isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4.18c0081b1c7bb2df namespace: kserve-ci-e2e-test resourceVersion: "18067" uid: 217a925b-f5f0-4b2c-9ed2-61fcb8032d90 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-9938b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18031" uid: 462c4bc4-d4e1-4953-8550-167bffd10da7 kind: Event lastTimestamp: "2026-07-07T14:26:04Z" message: Scaled up replica set isvc-sklearn-scale-raw-9938b-predictor-7c66754bb4 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:26:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:26:04Z" name: isvc-sklearn-scale-raw-9938b-predictor.18c0081b0fb76112 namespace: kserve-ci-e2e-test resourceVersion: "18034" uid: eb504bb7-00fd-447b-aba0-c6ec6301ce94 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:26:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-9938b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18032" uid: 28f2d7a9-e6f5-4ff6-b077-8e09e224c138 kind: Event lastTimestamp: "2026-07-07T14:27:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:26:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:27:04Z" name: isvc-sklearn-scale-raw-9938b-predictor.18c0081e8e1f67c9 namespace: kserve-ci-e2e-test resourceVersion: "18554" uid: fe29bc45-9b60-489c-8195-9f1cda0ad2ad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:26:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-9938b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18032" uid: 28f2d7a9-e6f5-4ff6-b077-8e09e224c138 kind: Event lastTimestamp: "2026-07-07T14:27:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:26:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:27:04Z" name: isvc-sklearn-scale-raw-9938b-predictor.18c0081e8e901854 namespace: kserve-ci-e2e-test resourceVersion: "18555" uid: a3035926-7ba1-42b9-8950-2f5082ec81ae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:27:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-9938b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18233" uid: 28f2d7a9-e6f5-4ff6-b077-8e09e224c138 kind: Event lastTimestamp: "2026-07-07T14:28:34Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:28:34Z" name: isvc-sklearn-scale-raw-9938b-predictor.18c0082c8b698627 namespace: kserve-ci-e2e-test resourceVersion: "19189" uid: a552488e-92e3-43b2-a2b8-a592b312bc83 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:27:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-9938b-predictor namespace: kserve-ci-e2e-test resourceVersion: "18233" uid: 28f2d7a9-e6f5-4ff6-b077-8e09e224c138 kind: Event lastTimestamp: "2026-07-07T14:28:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:27:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:28:35Z" name: isvc-sklearn-scale-raw-9938b-predictor.18c0082c8bdaa16a namespace: kserve-ci-e2e-test resourceVersion: "19190" uid: 3023353c-e6df-4b76-bd87-3d3f2dc2c7db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T14:26:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-9938b namespace: kserve-ci-e2e-test resourceVersion: "18021" uid: 87c97208-b528-4ef5-83ba-06fa6e12bf3a kind: Event lastTimestamp: "2026-07-07T14:28:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:26:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:28:41Z" name: isvc-sklearn-scale-raw-9938b.18c0081b0fb1eb9a namespace: kserve-ci-e2e-test resourceVersion: "19237" uid: f944815a-a0d2-437f-8920-bff783399530 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-9938b namespace: kserve-ci-e2e-test resourceVersion: "18021" uid: 87c97208-b528-4ef5-83ba-06fa6e12bf3a kind: Event lastTimestamp: "2026-07-07T14:26:04Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-9938b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-9938b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:26:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:26:04Z" name: isvc-sklearn-scale-raw-9938b.18c0081b109934ee namespace: kserve-ci-e2e-test resourceVersion: "18037" uid: 0ec78775-fbf9-407b-b3bf-51c941eff0f8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:28:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-9938b namespace: kserve-ci-e2e-test resourceVersion: "19235" uid: 87c97208-b528-4ef5-83ba-06fa6e12bf3a kind: Event lastTimestamp: "2026-07-07T14:28:41Z" message: InferenceService [isvc-sklearn-scale-raw-9938b] is Ready metadata: creationTimestamp: "2026-07-07T14:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:28:41Z" name: isvc-sklearn-scale-raw-9938b.18c0083f7c8d76c8 namespace: kserve-ci-e2e-test resourceVersion: "19236" uid: f635a678-996d-445d-b64b-6426644bd1df reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:20:28.999436Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: a03cc6cd-e0fc-4959-866a-b7d7d3d13cd1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc to ip-10-0-140-100.ec2.internal metadata: creationTimestamp: "2026-07-07T14:20:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:20:29Z" name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc.18c007cce2db01fb namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 65fc999e-31ec-4758-8f5d-b4a9203a513f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84448d6d9-ddnp2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:29Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc namespace: kserve-ci-e2e-test resourceVersion: "15025" uid: a03cc6cd-e0fc-4959-866a-b7d7d3d13cd1 kind: Event lastTimestamp: "2026-07-07T14:20:29Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:20:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:20:29Z" name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc.18c007ccfc77a617 namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: b4152124-6b28-4477-aa71-82b9cb8abd88 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: a03cc6cd-e0fc-4959-866a-b7d7d3d13cd1 kind: Event lastTimestamp: "2026-07-07T14:20:29Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:20:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:29Z" name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc.18c007ccfe19f230 namespace: kserve-ci-e2e-test resourceVersion: "15046" uid: 0c62573f-7b84-43f5-8cd6-7e37f5a66a22 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: a03cc6cd-e0fc-4959-866a-b7d7d3d13cd1 kind: Event lastTimestamp: "2026-07-07T14:20:29Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:20:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:29Z" name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc.18c007cd010f4c7f namespace: kserve-ci-e2e-test resourceVersion: "15047" uid: 5a1fc48f-8429-48f9-a1f3-0fed2906b97e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:29Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: a03cc6cd-e0fc-4959-866a-b7d7d3d13cd1 kind: Event lastTimestamp: "2026-07-07T14:20:29Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:20:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:29Z" name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc.18c007cd0180e49a namespace: kserve-ci-e2e-test resourceVersion: "15048" uid: b06d30e4-71fe-4e5b-913c-75a29d8f43c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: a03cc6cd-e0fc-4959-866a-b7d7d3d13cd1 kind: Event lastTimestamp: "2026-07-07T14:20:33Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-07T14:20:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:33Z" name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc.18c007cdf81cd027 namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: 5ebdefd7-7b4c-4909-a0cc-68b20382a43b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: a03cc6cd-e0fc-4959-866a-b7d7d3d13cd1 kind: Event lastTimestamp: "2026-07-07T14:20:54Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 20.585s (20.585s including waiting). Image size: 1305547511 bytes.' metadata: creationTimestamp: "2026-07-07T14:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:54Z" name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc.18c007d2c319f3e3 namespace: kserve-ci-e2e-test resourceVersion: "15260" uid: efdb7c1c-f00f-45f6-95d9-632ab4d57920 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: a03cc6cd-e0fc-4959-866a-b7d7d3d13cd1 kind: Event lastTimestamp: "2026-07-07T14:20:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:54Z" name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc.18c007d2c6819b56 namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 0ec9610d-65c0-4131-b935-b42d762b6278 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: a03cc6cd-e0fc-4959-866a-b7d7d3d13cd1 kind: Event lastTimestamp: "2026-07-07T14:20:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:54Z" name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc.18c007d2c73472a7 namespace: kserve-ci-e2e-test resourceVersion: "15262" uid: 89eed952-ae61-46da-b31c-8d8679871441 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: a03cc6cd-e0fc-4959-866a-b7d7d3d13cd1 kind: Event lastTimestamp: "2026-07-07T14:20:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:54Z" name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc.18c007d2c73fd53c namespace: kserve-ci-e2e-test resourceVersion: "15263" uid: 963641b6-ef4c-4a54-9ea1-9e10b57e6576 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: a03cc6cd-e0fc-4959-866a-b7d7d3d13cd1 kind: Event lastTimestamp: "2026-07-07T14:20:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:54Z" name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc.18c007d2ca502d22 namespace: kserve-ci-e2e-test resourceVersion: "15265" uid: 75ac0731-302b-482b-bcbe-9645f3a191eb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: a03cc6cd-e0fc-4959-866a-b7d7d3d13cd1 kind: Event lastTimestamp: "2026-07-07T14:20:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:20:54Z" name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc.18c007d2cabfd2cc namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 5a2d66dc-2670-44dc-a1f4-28c8df84c8a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-07T14:20:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: a03cc6cd-e0fc-4959-866a-b7d7d3d13cd1 kind: Event lastTimestamp: "2026-07-07T14:22:21Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:20:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:21Z" name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc.18c007d31de56d3a namespace: kserve-ci-e2e-test resourceVersion: "16108" uid: 4e6e1882-bc71-4049-aa81-229f6a31cee0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: a03cc6cd-e0fc-4959-866a-b7d7d3d13cd1 kind: Event lastTimestamp: "2026-07-07T14:22:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:22:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:19Z" name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc.18c007e6a0cfe498 namespace: kserve-ci-e2e-test resourceVersion: "16061" uid: f492eede-28a4-4a56-86ce-8c125f565665 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: a03cc6cd-e0fc-4959-866a-b7d7d3d13cd1 kind: Event lastTimestamp: "2026-07-07T14:22:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:22:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:19Z" name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc.18c007e6a0d08882 namespace: kserve-ci-e2e-test resourceVersion: "16062" uid: 95e75f95-87e1-4659-b868-ae2ff1211eeb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: a03cc6cd-e0fc-4959-866a-b7d7d3d13cd1 kind: Event lastTimestamp: "2026-07-07T14:22:21Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:21Z" name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc.18c007e724112bc4 namespace: kserve-ci-e2e-test resourceVersion: "16107" uid: 59b0ea2c-6847-4483-8a1b-a37ba1b3a00b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589 namespace: kserve-ci-e2e-test resourceVersion: "15002" uid: e92fe7f1-7af7-4274-bdbe-95ebca31c9a1 kind: Event lastTimestamp: "2026-07-07T14:20:28Z" message: 'Created pod: isvc-xgboost-graph-raw-884de-predictor-867f8cd589-k9mcc' metadata: creationTimestamp: "2026-07-07T14:20:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:20:28Z" name: isvc-xgboost-graph-raw-884de-predictor-867f8cd589.18c007cce21ff5e0 namespace: kserve-ci-e2e-test resourceVersion: "15015" uid: 540cd2c0-da46-4bd3-99fd-43c62636877d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:20:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-884de-predictor namespace: kserve-ci-e2e-test resourceVersion: "14999" uid: 89d66360-ddbb-46ea-b7f1-174efe233613 kind: Event lastTimestamp: "2026-07-07T14:20:28Z" message: Scaled up replica set isvc-xgboost-graph-raw-884de-predictor-867f8cd589 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:20:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:20:28Z" name: isvc-xgboost-graph-raw-884de-predictor.18c007ccdef8cd33 namespace: kserve-ci-e2e-test resourceVersion: "15006" uid: edb059d5-7dda-4855-9fa0-ee7a8bc38366 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:20:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-884de-predictor namespace: kserve-ci-e2e-test resourceVersion: "15001" uid: 0a73f664-a444-447f-9018-396156faf993 kind: Event lastTimestamp: "2026-07-07T14:21:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:20:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:21:29Z" name: isvc-xgboost-graph-raw-884de-predictor.18c007d05e06f3a7 namespace: kserve-ci-e2e-test resourceVersion: "15534" uid: 561d12d8-112f-4a80-b3cd-2d4d5d7519e8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:20:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-884de-predictor namespace: kserve-ci-e2e-test resourceVersion: "15001" uid: 0a73f664-a444-447f-9018-396156faf993 kind: Event lastTimestamp: "2026-07-07T14:21:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:20:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:21:29Z" name: isvc-xgboost-graph-raw-884de-predictor.18c007d05e817d26 namespace: kserve-ci-e2e-test resourceVersion: "15536" uid: 082a2152-e8d8-4d1f-be51-5a245debed1e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:21:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-884de-predictor namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 0a73f664-a444-447f-9018-396156faf993 kind: Event lastTimestamp: "2026-07-07T14:22:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:22:14Z" name: isvc-xgboost-graph-raw-884de-predictor.18c007de5b898bda namespace: kserve-ci-e2e-test resourceVersion: "15904" uid: f0890484-9c2b-44bc-84b0-a60fff4a46c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:21:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-884de-predictor namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 0a73f664-a444-447f-9018-396156faf993 kind: Event lastTimestamp: "2026-07-07T14:22:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:21:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:22:14Z" name: isvc-xgboost-graph-raw-884de-predictor.18c007de5c0a0d43 namespace: kserve-ci-e2e-test resourceVersion: "15905" uid: b92ca9bb-8ec0-4673-bb3b-d51f08b64306 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T14:20:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-884de namespace: kserve-ci-e2e-test resourceVersion: "14977" uid: e4ffd8f5-7675-4d6d-a060-1c0c3c13fabe kind: Event lastTimestamp: "2026-07-07T14:22:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:20:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:22:01Z" name: isvc-xgboost-graph-raw-884de.18c007ccdf4177ff namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: 361c39b7-b8d3-4fd9-bb9e-73960d6afce2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-884de namespace: kserve-ci-e2e-test resourceVersion: "15779" uid: e4ffd8f5-7675-4d6d-a060-1c0c3c13fabe kind: Event lastTimestamp: "2026-07-07T14:22:01Z" message: InferenceService [isvc-xgboost-graph-raw-884de] is Ready metadata: creationTimestamp: "2026-07-07T14:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:22:01Z" name: isvc-xgboost-graph-raw-884de.18c007e282677429 namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: cccd9b3a-e463-4d05-bfa7-94a4f3cc9a98 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:22:19.450309Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: e95a58c6-8d8c-4249-80db-35ef7ca89c2d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz to ip-10-0-140-100.ec2.internal metadata: creationTimestamp: "2026-07-07T14:22:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:22:19Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz.18c007e69a3dd8f3 namespace: kserve-ci-e2e-test resourceVersion: "16040" uid: 4d953500-e274-4dcd-a5ad-e2585ac7484d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84448d6d9-ddnp2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:19Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz namespace: kserve-ci-e2e-test resourceVersion: "16057" uid: e95a58c6-8d8c-4249-80db-35ef7ca89c2d kind: Event lastTimestamp: "2026-07-07T14:22:19Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:22:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:22:19Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz.18c007e6b3f3e4b9 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 3757acc0-65ac-4479-baba-025826f1e363 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: e95a58c6-8d8c-4249-80db-35ef7ca89c2d kind: Event lastTimestamp: "2026-07-07T14:22:19Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:22:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:19Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz.18c007e6b5ee3cc2 namespace: kserve-ci-e2e-test resourceVersion: "16076" uid: a36fa5a5-5ea5-44eb-a479-e3de394a5c66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: e95a58c6-8d8c-4249-80db-35ef7ca89c2d kind: Event lastTimestamp: "2026-07-07T14:22:19Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:22:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:19Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz.18c007e6b9134829 namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: 35956dda-fd48-4c3c-a439-7bdf1f1195c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: e95a58c6-8d8c-4249-80db-35ef7ca89c2d kind: Event lastTimestamp: "2026-07-07T14:22:19Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:22:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:19Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz.18c007e6b9c282a6 namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: 00ce6154-cdc0-497c-b102-6fc0a796fbf9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: e95a58c6-8d8c-4249-80db-35ef7ca89c2d kind: Event lastTimestamp: "2026-07-07T14:22:24Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-07-07T14:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:24Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz.18c007e7b1f39398 namespace: kserve-ci-e2e-test resourceVersion: "16130" uid: 7812fe28-8de0-4688-aa01-03e5b210a44a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: e95a58c6-8d8c-4249-80db-35ef7ca89c2d kind: Event lastTimestamp: "2026-07-07T14:22:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:24Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz.18c007e7b4966db4 namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: 9bf3d10a-b00d-4ecf-883b-3b218b2d48ea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: e95a58c6-8d8c-4249-80db-35ef7ca89c2d kind: Event lastTimestamp: "2026-07-07T14:22:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:24Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz.18c007e7b51d618b namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: c587945b-d376-4ba3-934b-d223ef3b56e6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: e95a58c6-8d8c-4249-80db-35ef7ca89c2d kind: Event lastTimestamp: "2026-07-07T14:22:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:24Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz.18c007e7b5268056 namespace: kserve-ci-e2e-test resourceVersion: "16142" uid: 74bfdab2-3c37-49c3-a7a9-2e011059bdcc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: e95a58c6-8d8c-4249-80db-35ef7ca89c2d kind: Event lastTimestamp: "2026-07-07T14:22:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:24Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz.18c007e7b7d2eae7 namespace: kserve-ci-e2e-test resourceVersion: "16143" uid: 611f359a-1615-4221-9a4e-8489b67e2217 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: e95a58c6-8d8c-4249-80db-35ef7ca89c2d kind: Event lastTimestamp: "2026-07-07T14:22:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:24Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz.18c007e7b8427035 namespace: kserve-ci-e2e-test resourceVersion: "16146" uid: e1f9669f-41e0-4fc2-a52c-42bf24564bf8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-07T14:22:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: e95a58c6-8d8c-4249-80db-35ef7ca89c2d kind: Event lastTimestamp: "2026-07-07T14:24:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:22:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:12Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz.18c007e829e4f921 namespace: kserve-ci-e2e-test resourceVersion: "17109" uid: df3e317f-5ad1-4606-9093-03abff91e264 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: e95a58c6-8d8c-4249-80db-35ef7ca89c2d kind: Event lastTimestamp: "2026-07-07T14:24:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:09Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz.18c008003bccdf0a namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: feb27162-6df1-45e9-9836-4a5776795aba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: e95a58c6-8d8c-4249-80db-35ef7ca89c2d kind: Event lastTimestamp: "2026-07-07T14:24:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:09Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz.18c008003bcd7c6c namespace: kserve-ci-e2e-test resourceVersion: "17078" uid: 6e4ea046-1289-4198-91e8-6f28cc73a292 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: e95a58c6-8d8c-4249-80db-35ef7ca89c2d kind: Event lastTimestamp: "2026-07-07T14:24:12Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:24:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:12Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz.18c00800d84ed302 namespace: kserve-ci-e2e-test resourceVersion: "17107" uid: 561d4773-3167-4d89-b019-66cc26780478 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9 namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: 2d7cb0f9-3b5b-4164-9027-dbbad1fb4a68 kind: Event lastTimestamp: "2026-07-07T14:22:19Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9-slvbz' metadata: creationTimestamp: "2026-07-07T14:22:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:22:19Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9.18c007e699c11f1f namespace: kserve-ci-e2e-test resourceVersion: "16036" uid: e1fd8484-bbe1-4f17-8f09-a2cb207c582a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-bd33a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16021" uid: a490315e-ebdf-45ea-9890-8ea46aef581d kind: Event lastTimestamp: "2026-07-07T14:22:19Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-bd33a-predictor-6d98dbdfd9 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:22:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:22:19Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor.18c007e692cf5006 namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 2a44e7bb-9d49-4bbb-b75d-c37cb0829513 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-bd33a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16023" uid: 7cf4ddb2-077d-4555-93f7-d36d3bce9568 kind: Event lastTimestamp: "2026-07-07T14:22:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:22:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:22:34Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor.18c007ea114dcebb namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: 76f7cf27-9c61-4602-be5b-8ca6647dc8ba reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-bd33a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16023" uid: 7cf4ddb2-077d-4555-93f7-d36d3bce9568 kind: Event lastTimestamp: "2026-07-07T14:22:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:22:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:22:34Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor.18c007ea11bb8e52 namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 0e2f620c-9036-4096-8067-c94fd06e5b87 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T14:22:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-bd33a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 7cf4ddb2-077d-4555-93f7-d36d3bce9568 kind: Event lastTimestamp: "2026-07-07T14:23:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:22:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:49Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor.18c007ed908e2ca6 namespace: kserve-ci-e2e-test resourceVersion: "16811" uid: 59cd7b12-79e7-4972-9b1c-beaa0a54b97d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-07T14:22:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-bd33a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 7cf4ddb2-077d-4555-93f7-d36d3bce9568 kind: Event lastTimestamp: "2026-07-07T14:23:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:22:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:49Z" name: isvc-xgboost-graph-raw-hpa-bd33a-predictor.18c007ed914cb507 namespace: kserve-ci-e2e-test resourceVersion: "16812" uid: e9cd3e21-973b-4d35-9696-6c59233fb952 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-07T14:22:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-bd33a namespace: kserve-ci-e2e-test resourceVersion: "15996" uid: e930f51d-c9b0-4cae-b60b-22e977c446cc kind: Event lastTimestamp: "2026-07-07T14:23:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:22:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:23:32Z" name: isvc-xgboost-graph-raw-hpa-bd33a.18c007e692954c00 namespace: kserve-ci-e2e-test resourceVersion: "16677" uid: 664a6d19-6a82-4983-972b-4c4564daf8e5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-bd33a namespace: kserve-ci-e2e-test resourceVersion: "16675" uid: e930f51d-c9b0-4cae-b60b-22e977c446cc kind: Event lastTimestamp: "2026-07-07T14:23:32Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-bd33a] is Ready metadata: creationTimestamp: "2026-07-07T14:23:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:23:32Z" name: isvc-xgboost-graph-raw-hpa-bd33a.18c007f78e6af54e namespace: kserve-ci-e2e-test resourceVersion: "16676" uid: 5f4b509f-bb0d-434e-9c37-ef7a3ddbdb84 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:24:09.456077Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv namespace: kserve-ci-e2e-test resourceVersion: "17036" uid: 0e9de898-2676-403c-93b6-dcdc93af1c6c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-7a71c-predictor-d86df68f-qtdkv to ip-10-0-139-136.ec2.internal metadata: creationTimestamp: "2026-07-07T14:24:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:24:09Z" name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv.18c00800371868a4 namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: f462facc-68c9-4c1c-bec2-bb9ea492c102 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84448d6d9-ddnp2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:09Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv namespace: kserve-ci-e2e-test resourceVersion: "17065" uid: 0e9de898-2676-403c-93b6-dcdc93af1c6c kind: Event lastTimestamp: "2026-07-07T14:24:09Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:24:09Z" name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv.18c0080050f82b70 namespace: kserve-ci-e2e-test resourceVersion: "17084" uid: 187d424d-b084-41f6-8858-13e49946889f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: 0e9de898-2676-403c-93b6-dcdc93af1c6c kind: Event lastTimestamp: "2026-07-07T14:24:09Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-07T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:09Z" name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv.18c0080052c35937 namespace: kserve-ci-e2e-test resourceVersion: "17086" uid: 9ec04ee3-ffd7-439f-8892-9bd3a93c8d23 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: 0e9de898-2676-403c-93b6-dcdc93af1c6c kind: Event lastTimestamp: "2026-07-07T14:24:11Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.246s (1.246s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-07T14:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:11Z" name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv.18c008009d15e4aa namespace: kserve-ci-e2e-test resourceVersion: "17097" uid: 6ab0d26a-1286-4a06-87be-240c175b315e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: 0e9de898-2676-403c-93b6-dcdc93af1c6c kind: Event lastTimestamp: "2026-07-07T14:24:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:11Z" name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv.18c00800a08e3eb2 namespace: kserve-ci-e2e-test resourceVersion: "17098" uid: 9acbe853-7bd7-4b93-9016-ee36472d6694 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: 0e9de898-2676-403c-93b6-dcdc93af1c6c kind: Event lastTimestamp: "2026-07-07T14:24:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:11Z" name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv.18c00800a109707f namespace: kserve-ci-e2e-test resourceVersion: "17099" uid: 138a3820-7722-4425-a478-db5aa6471338 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: 0e9de898-2676-403c-93b6-dcdc93af1c6c kind: Event lastTimestamp: "2026-07-07T14:24:11Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-07T14:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:11Z" name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv.18c00800a118a2eb namespace: kserve-ci-e2e-test resourceVersion: "17100" uid: 29b3ebd3-8c7b-4c49-b454-155d32014cb5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: 0e9de898-2676-403c-93b6-dcdc93af1c6c kind: Event lastTimestamp: "2026-07-07T14:24:13Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.528s (2.528s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-07T14:24:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:13Z" name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv.18c0080137cbabe0 namespace: kserve-ci-e2e-test resourceVersion: "17128" uid: 6459e350-59a1-4b71-a961-750dd2536df5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: 0e9de898-2676-403c-93b6-dcdc93af1c6c kind: Event lastTimestamp: "2026-07-07T14:24:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:24:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:13Z" name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv.18c008013b1ff12d namespace: kserve-ci-e2e-test resourceVersion: "17129" uid: 42c091f0-267d-49be-8772-1aa82b42f582 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: 0e9de898-2676-403c-93b6-dcdc93af1c6c kind: Event lastTimestamp: "2026-07-07T14:24:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:24:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:13Z" name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv.18c008013b99cf7b namespace: kserve-ci-e2e-test resourceVersion: "17130" uid: 903652ca-2511-487b-89b0-ac861e87398e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: 0e9de898-2676-403c-93b6-dcdc93af1c6c kind: Event lastTimestamp: "2026-07-07T14:26:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:05Z" name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv.18c0081b20b4ab84 namespace: kserve-ci-e2e-test resourceVersion: "18088" uid: 845a2f2f-845b-4d65-9b01-5bac036b5df6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:26:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: 0e9de898-2676-403c-93b6-dcdc93af1c6c kind: Event lastTimestamp: "2026-07-07T14:26:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:26:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:26:05Z" name: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv.18c0081b20b513f0 namespace: kserve-ci-e2e-test resourceVersion: "18091" uid: e48e99d9-8598-43c5-80c1-6d5e94c7020a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-7a71c-predictor-d86df68f namespace: kserve-ci-e2e-test resourceVersion: "17021" uid: 133e7c1a-ead5-4007-ab2e-d3a4df49ac44 kind: Event lastTimestamp: "2026-07-07T14:24:09Z" message: 'Created pod: message-dumper-raw-7a71c-predictor-d86df68f-qtdkv' metadata: creationTimestamp: "2026-07-07T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:09Z" name: message-dumper-raw-7a71c-predictor-d86df68f.18c0080036678437 namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: 2ce31f63-5d32-45e2-8075-593d2eb6cba6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-7a71c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17019" uid: 0af888fa-a629-4a74-975e-1ca46eef4167 kind: Event lastTimestamp: "2026-07-07T14:24:09Z" message: Scaled up replica set message-dumper-raw-7a71c-predictor-d86df68f from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:09Z" name: message-dumper-raw-7a71c-predictor.18c00800330ca6b4 namespace: kserve-ci-e2e-test resourceVersion: "17023" uid: 521dd32e-c0ab-43bc-b53d-6b49683e56a6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-7a71c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: 28d1e06b-e393-49e2-bd81-6c7476e4d34c kind: Event lastTimestamp: "2026-07-07T14:24:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:24:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:24Z" name: message-dumper-raw-7a71c-predictor.18c00803b164035a namespace: kserve-ci-e2e-test resourceVersion: "17218" uid: e5a9daf7-7b21-49ab-9840-a6a8e68c28d2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-7a71c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: 28d1e06b-e393-49e2-bd81-6c7476e4d34c kind: Event lastTimestamp: "2026-07-07T14:24:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:24:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:24Z" name: message-dumper-raw-7a71c-predictor.18c00803b1c62118 namespace: kserve-ci-e2e-test resourceVersion: "17220" uid: 9b7cfeb1-d687-4ca4-b479-bac7b2b1486b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-7a71c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17219" uid: 28d1e06b-e393-49e2-bd81-6c7476e4d34c kind: Event lastTimestamp: "2026-07-07T14:24:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:39Z" name: message-dumper-raw-7a71c-predictor.18c0080730f73b49 namespace: kserve-ci-e2e-test resourceVersion: "17396" uid: e87da2d3-2365-4268-ab6a-5d01ada019d4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-7a71c-predictor namespace: kserve-ci-e2e-test resourceVersion: "17219" uid: 28d1e06b-e393-49e2-bd81-6c7476e4d34c kind: Event lastTimestamp: "2026-07-07T14:24:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:24:39Z" name: message-dumper-raw-7a71c-predictor.18c00807316c0d63 namespace: kserve-ci-e2e-test resourceVersion: "17399" uid: bd8552fe-2b14-4f88-8bd6-4809b9bc9abe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-07T14:24:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-7a71c namespace: kserve-ci-e2e-test resourceVersion: "17009" uid: 3f896ee8-3068-41df-bf62-e397d74f89e8 kind: Event lastTimestamp: "2026-07-07T14:24:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:21Z" name: message-dumper-raw-7a71c.18c0080032e16c87 namespace: kserve-ci-e2e-test resourceVersion: "17199" uid: 32eee066-c525-4373-b334-6e055a9a4f36 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-7a71c namespace: kserve-ci-e2e-test resourceVersion: "17009" uid: 3f896ee8-3068-41df-bf62-e397d74f89e8 kind: Event lastTimestamp: "2026-07-07T14:24:09Z" message: 'Failed to update status for InferenceService "message-dumper-raw-7a71c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-7a71c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:09Z" name: message-dumper-raw-7a71c.18c0080033305dda namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 2698ca0d-0c3f-4a83-9984-266265dced63 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-7a71c namespace: kserve-ci-e2e-test resourceVersion: "17196" uid: 3f896ee8-3068-41df-bf62-e397d74f89e8 kind: Event lastTimestamp: "2026-07-07T14:24:21Z" message: InferenceService [message-dumper-raw-7a71c] is Ready metadata: creationTimestamp: "2026-07-07T14:24:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:24:21Z" name: message-dumper-raw-7a71c.18c008031d672067 namespace: kserve-ci-e2e-test resourceVersion: "17197" uid: 7782b907-b91f-4807-b70c-0c1d91f3c159 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:22:08.868988Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-884de-5f6f5ccc9d-gjgd9 namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: 616be206-f6bc-4db5-8a4a-b72e6375b8d7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-884de-5f6f5ccc9d-gjgd9 to ip-10-0-139-136.ec2.internal metadata: creationTimestamp: "2026-07-07T14:22:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:22:08Z" name: model-chainer-raw-884de-5f6f5ccc9d-gjgd9.18c007e4238b70fd namespace: kserve-ci-e2e-test resourceVersion: "15846" uid: 02abd324-e34b-43d5-bdbd-852eaf66ec9b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84448d6d9-ddnp2 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:22:09Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-884de-5f6f5ccc9d-gjgd9 namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 616be206-f6bc-4db5-8a4a-b72e6375b8d7 kind: Event lastTimestamp: "2026-07-07T14:22:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-884de-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:22:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:09Z" name: model-chainer-raw-884de-5f6f5ccc9d-gjgd9.18c007e42c588e63 namespace: kserve-ci-e2e-test resourceVersion: "15859" uid: 4d5a3af3-57b5-476b-92c5-1c9a7d98bfe6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:10Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-884de-5f6f5ccc9d-gjgd9 namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: 616be206-f6bc-4db5-8a4a-b72e6375b8d7 kind: Event lastTimestamp: "2026-07-07T14:22:10Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:22:10Z" name: model-chainer-raw-884de-5f6f5ccc9d-gjgd9.18c007e4966d4370 namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: beb2be2d-2aef-42fc-a1d9-3537b7ba02d4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-884de} kind: Pod name: model-chainer-raw-884de-5f6f5ccc9d-gjgd9 namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 616be206-f6bc-4db5-8a4a-b72e6375b8d7 kind: Event lastTimestamp: "2026-07-07T14:22:10Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:3bfaa94d16d20ed0bc6642496b738f1c154c88684ea41dc8df471bf4e0a2804c" metadata: creationTimestamp: "2026-07-07T14:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:10Z" name: model-chainer-raw-884de-5f6f5ccc9d-gjgd9.18c007e4980050bc namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: 0ff31c40-618c-4c8d-833f-a2df68510fce reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-884de} kind: Pod name: model-chainer-raw-884de-5f6f5ccc9d-gjgd9 namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 616be206-f6bc-4db5-8a4a-b72e6375b8d7 kind: Event lastTimestamp: "2026-07-07T14:22:13Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:3bfaa94d16d20ed0bc6642496b738f1c154c88684ea41dc8df471bf4e0a2804c" in 2.374s (2.374s including waiting). Image size: 218072702 bytes.' metadata: creationTimestamp: "2026-07-07T14:22:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:13Z" name: model-chainer-raw-884de-5f6f5ccc9d-gjgd9.18c007e5258dbfe7 namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 6fe3e636-234e-423e-8455-4244fabdd2ce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-884de} kind: Pod name: model-chainer-raw-884de-5f6f5ccc9d-gjgd9 namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 616be206-f6bc-4db5-8a4a-b72e6375b8d7 kind: Event lastTimestamp: "2026-07-07T14:22:13Z" message: 'Created container: model-chainer-raw-884de' metadata: creationTimestamp: "2026-07-07T14:22:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:13Z" name: model-chainer-raw-884de-5f6f5ccc9d-gjgd9.18c007e52871d2f0 namespace: kserve-ci-e2e-test resourceVersion: "15894" uid: e12c9345-a288-4864-b66a-53cd1ab99652 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-884de} kind: Pod name: model-chainer-raw-884de-5f6f5ccc9d-gjgd9 namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 616be206-f6bc-4db5-8a4a-b72e6375b8d7 kind: Event lastTimestamp: "2026-07-07T14:22:13Z" message: Started container model-chainer-raw-884de metadata: creationTimestamp: "2026-07-07T14:22:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:13Z" name: model-chainer-raw-884de-5f6f5ccc9d-gjgd9.18c007e528d46e9b namespace: kserve-ci-e2e-test resourceVersion: "15895" uid: 78bbddc5-20c9-4002-8de3-d89b5990319e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-884de} kind: Pod name: model-chainer-raw-884de-5f6f5ccc9d-gjgd9 namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 616be206-f6bc-4db5-8a4a-b72e6375b8d7 kind: Event lastTimestamp: "2026-07-07T14:22:18Z" message: Stopping container model-chainer-raw-884de metadata: creationTimestamp: "2026-07-07T14:22:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:18Z" name: model-chainer-raw-884de-5f6f5ccc9d-gjgd9.18c007e679eca272 namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: 91a80ac9-e330-46cc-a938-4c95431bb47d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:22:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-884de} kind: Pod name: model-chainer-raw-884de-5f6f5ccc9d-gjgd9 namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 616be206-f6bc-4db5-8a4a-b72e6375b8d7 kind: Event lastTimestamp: "2026-07-07T14:22:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T14:22:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:43Z" name: model-chainer-raw-884de-5f6f5ccc9d-gjgd9.18c007e67a62fe91 namespace: kserve-ci-e2e-test resourceVersion: "16307" uid: ffc707fc-3489-4057-b041-40486ec871e5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-884de} kind: Pod name: model-chainer-raw-884de-5f6f5ccc9d-gjgd9 namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 616be206-f6bc-4db5-8a4a-b72e6375b8d7 kind: Event lastTimestamp: "2026-07-07T14:22:48Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8080/readyz": dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:22:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:22:48Z" name: model-chainer-raw-884de-5f6f5ccc9d-gjgd9.18c007ed764f4229 namespace: kserve-ci-e2e-test resourceVersion: "16342" uid: f48458ff-da95-41fa-b0a8-c902fd77c392 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-884de-5f6f5ccc9d namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 7169e0d4-f446-4dee-ad85-983147310645 kind: Event lastTimestamp: "2026-07-07T14:22:08Z" message: 'Created pod: model-chainer-raw-884de-5f6f5ccc9d-gjgd9' metadata: creationTimestamp: "2026-07-07T14:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:22:08Z" name: model-chainer-raw-884de-5f6f5ccc9d.18c007e421c04600 namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: f48b499d-7e9b-4be2-9a27-f10ef292d01f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-884de namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: 19f1e1e0-a283-4aea-9b37-8c1f12963f4e kind: Event lastTimestamp: "2026-07-07T14:22:08Z" message: Scaled up replica set model-chainer-raw-884de-5f6f5ccc9d from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:22:08Z" name: model-chainer-raw-884de.18c007e41ff4846e namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 41b214d9-0e26-48d6-99ec-5bcc144857b8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:22:08Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-884de namespace: kserve-ci-e2e-test resourceVersion: "15851" uid: eb11fb18-e92b-4436-8f39-4b48a3ca8989 kind: Event lastTimestamp: "2026-07-07T14:22:08Z" message: InferenceGraph [model-chainer-raw-884de] is Ready metadata: creationTimestamp: "2026-07-07T14:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:22:08Z" name: model-chainer-raw-884de.18c007e428581378 namespace: kserve-ci-e2e-test resourceVersion: "15853" uid: cd1d5d2f-4d94-4c09-b5ab-10664a1caf85 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:23:59.186368Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-bd33a-64d55c4c48-8w8pz namespace: kserve-ci-e2e-test resourceVersion: "16892" uid: 6bb16c97-eedf-4179-b199-4d47dee94551 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-bd33a-64d55c4c48-8w8pz to ip-10-0-139-136.ec2.internal metadata: creationTimestamp: "2026-07-07T14:23:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:23:59Z" name: model-chainer-raw-hpa-bd33a-64d55c4c48-8w8pz.18c007fdd2f913b1 namespace: kserve-ci-e2e-test resourceVersion: "16897" uid: 69ebdf8f-2961-4a3c-99f8-edf065df16b8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84448d6d9-ddnp2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:59Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-bd33a-64d55c4c48-8w8pz namespace: kserve-ci-e2e-test resourceVersion: "16901" uid: 6bb16c97-eedf-4179-b199-4d47dee94551 kind: Event lastTimestamp: "2026-07-07T14:23:59Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:23:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:23:59Z" name: model-chainer-raw-hpa-bd33a-64d55c4c48-8w8pz.18c007fdec9af3e9 namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: beb1e570-0601-48c8-be08-30aef572b222 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-bd33a} kind: Pod name: model-chainer-raw-hpa-bd33a-64d55c4c48-8w8pz namespace: kserve-ci-e2e-test resourceVersion: "16895" uid: 6bb16c97-eedf-4179-b199-4d47dee94551 kind: Event lastTimestamp: "2026-07-07T14:23:59Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3bfaa94d16d20ed0bc6642496b738f1c154c88684ea41dc8df471bf4e0a2804c" already present on machine metadata: creationTimestamp: "2026-07-07T14:23:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:59Z" name: model-chainer-raw-hpa-bd33a-64d55c4c48-8w8pz.18c007fdede290ca namespace: kserve-ci-e2e-test resourceVersion: "16913" uid: 5f491847-fa82-4dff-bfe9-11ccf283e2b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-bd33a} kind: Pod name: model-chainer-raw-hpa-bd33a-64d55c4c48-8w8pz namespace: kserve-ci-e2e-test resourceVersion: "16895" uid: 6bb16c97-eedf-4179-b199-4d47dee94551 kind: Event lastTimestamp: "2026-07-07T14:23:59Z" message: 'Created container: model-chainer-raw-hpa-bd33a' metadata: creationTimestamp: "2026-07-07T14:23:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:59Z" name: model-chainer-raw-hpa-bd33a-64d55c4c48-8w8pz.18c007fdf0d588f9 namespace: kserve-ci-e2e-test resourceVersion: "16914" uid: 80292879-510a-4b34-9055-d7ced6228c30 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-bd33a} kind: Pod name: model-chainer-raw-hpa-bd33a-64d55c4c48-8w8pz namespace: kserve-ci-e2e-test resourceVersion: "16895" uid: 6bb16c97-eedf-4179-b199-4d47dee94551 kind: Event lastTimestamp: "2026-07-07T14:23:59Z" message: Started container model-chainer-raw-hpa-bd33a metadata: creationTimestamp: "2026-07-07T14:23:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:23:59Z" name: model-chainer-raw-hpa-bd33a-64d55c4c48-8w8pz.18c007fdf14ba45a namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: 34fac095-5962-45e3-8aeb-14a4e686efc2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:24:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-bd33a} kind: Pod name: model-chainer-raw-hpa-bd33a-64d55c4c48-8w8pz namespace: kserve-ci-e2e-test resourceVersion: "16895" uid: 6bb16c97-eedf-4179-b199-4d47dee94551 kind: Event lastTimestamp: "2026-07-07T14:24:09Z" message: Stopping container model-chainer-raw-hpa-bd33a metadata: creationTimestamp: "2026-07-07T14:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:09Z" name: model-chainer-raw-hpa-bd33a-64d55c4c48-8w8pz.18c008002b3d704a namespace: kserve-ci-e2e-test resourceVersion: "17004" uid: 2deb3e5f-1d43-4b3e-b17f-8be902314888 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-07T14:24:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-bd33a} kind: Pod name: model-chainer-raw-hpa-bd33a-64d55c4c48-8w8pz namespace: kserve-ci-e2e-test resourceVersion: "16895" uid: 6bb16c97-eedf-4179-b199-4d47dee94551 kind: Event lastTimestamp: "2026-07-07T14:24:35Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-07T14:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:24:35Z" name: model-chainer-raw-hpa-bd33a-64d55c4c48-8w8pz.18c0080083fb8b12 namespace: kserve-ci-e2e-test resourceVersion: "17367" uid: 48d9b5ad-5087-44e3-a010-6fa6d50d4e4e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-136.ec2.internal source: component: kubelet host: ip-10-0-139-136.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-bd33a-64d55c4c48 namespace: kserve-ci-e2e-test resourceVersion: "16889" uid: a017d9b5-014b-4c98-8de8-e294e0648140 kind: Event lastTimestamp: "2026-07-07T14:23:59Z" message: 'Created pod: model-chainer-raw-hpa-bd33a-64d55c4c48-8w8pz' metadata: creationTimestamp: "2026-07-07T14:23:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:59Z" name: model-chainer-raw-hpa-bd33a-64d55c4c48.18c007fdd291c971 namespace: kserve-ci-e2e-test resourceVersion: "16894" uid: cb304a78-d6b9-4d09-bc52-5388078ee59f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-bd33a namespace: kserve-ci-e2e-test resourceVersion: "16887" uid: 44c49a37-be85-46ae-8bd8-031f4f6aec8f kind: Event lastTimestamp: "2026-07-07T14:23:59Z" message: Scaled up replica set model-chainer-raw-hpa-bd33a-64d55c4c48 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:23:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:23:59Z" name: model-chainer-raw-hpa-bd33a.18c007fdd19ec113 namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: 9406354c-5597-4d50-a37f-c88cf27199bb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:23:59Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-bd33a namespace: kserve-ci-e2e-test resourceVersion: "16903" uid: e4906860-3396-419d-9f4c-2a87880ac13c kind: Event lastTimestamp: "2026-07-07T14:23:59Z" message: InferenceGraph [model-chainer-raw-hpa-bd33a] is Ready metadata: creationTimestamp: "2026-07-07T14:23:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:23:59Z" name: model-chainer-raw-hpa-bd33a.18c007fdd754bd43 namespace: kserve-ci-e2e-test resourceVersion: "16905" uid: 592648f9-ab18-418b-a62b-e6f908f02e67 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:30:56.718183Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-73688-predictor-65d96b6466-67vtr namespace: kserve-ci-e2e-test resourceVersion: "20488" uid: 0976311d-7f96-496a-8f90-2411fa4978f6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-73688-predictor-65d96b6466-67vtr to ip-10-0-140-100.ec2.internal metadata: creationTimestamp: "2026-07-07T14:30:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:30:56Z" name: raw-sklearn-73688-predictor-65d96b6466-67vtr.18c0085f09cef9d3 namespace: kserve-ci-e2e-test resourceVersion: "20494" uid: 4552a990-53c8-43cb-93d7-6cf25411b807 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84448d6d9-ddnp2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:57Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-73688-predictor-65d96b6466-67vtr namespace: kserve-ci-e2e-test resourceVersion: "20498" uid: 0976311d-7f96-496a-8f90-2411fa4978f6 kind: Event lastTimestamp: "2026-07-07T14:30:57Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:30:57Z" name: raw-sklearn-73688-predictor-65d96b6466-67vtr.18c0085f233d7fb3 namespace: kserve-ci-e2e-test resourceVersion: "20507" uid: cc5f72ca-5471-405c-9b89-c950a170baa6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-73688-predictor-65d96b6466-67vtr namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: 0976311d-7f96-496a-8f90-2411fa4978f6 kind: Event lastTimestamp: "2026-07-07T14:30:57Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:30:57Z" name: raw-sklearn-73688-predictor-65d96b6466-67vtr.18c0085f24a59f32 namespace: kserve-ci-e2e-test resourceVersion: "20509" uid: 5535b5a6-8e9c-48f8-aa4c-f02dd69b50c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-73688-predictor-65d96b6466-67vtr namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: 0976311d-7f96-496a-8f90-2411fa4978f6 kind: Event lastTimestamp: "2026-07-07T14:30:57Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:30:57Z" name: raw-sklearn-73688-predictor-65d96b6466-67vtr.18c0085f2772a5d7 namespace: kserve-ci-e2e-test resourceVersion: "20510" uid: 1440e64a-52a1-41a4-9487-f0e76da97b60 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-73688-predictor-65d96b6466-67vtr namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: 0976311d-7f96-496a-8f90-2411fa4978f6 kind: Event lastTimestamp: "2026-07-07T14:30:57Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:30:57Z" name: raw-sklearn-73688-predictor-65d96b6466-67vtr.18c0085f27e0e449 namespace: kserve-ci-e2e-test resourceVersion: "20511" uid: 596961f6-cc5d-4749-8c41-d2c842f9917d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-73688-predictor-65d96b6466-67vtr namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: 0976311d-7f96-496a-8f90-2411fa4978f6 kind: Event lastTimestamp: "2026-07-07T14:31:01Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:31:01Z" name: raw-sklearn-73688-predictor-65d96b6466-67vtr.18c0086017e290f3 namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: 1b2194e2-bcf1-471b-b510-857a072698aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-73688-predictor-65d96b6466-67vtr namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: 0976311d-7f96-496a-8f90-2411fa4978f6 kind: Event lastTimestamp: "2026-07-07T14:31:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:31:01Z" name: raw-sklearn-73688-predictor-65d96b6466-67vtr.18c008601b151e29 namespace: kserve-ci-e2e-test resourceVersion: "20548" uid: dd375c6e-fc0f-4e35-841c-2f1de3d6cc3b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-73688-predictor-65d96b6466-67vtr namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: 0976311d-7f96-496a-8f90-2411fa4978f6 kind: Event lastTimestamp: "2026-07-07T14:31:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:31:01Z" name: raw-sklearn-73688-predictor-65d96b6466-67vtr.18c008601b85e784 namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: b3690fbf-8d72-4e50-a1a7-dc73648aae22 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-73688-predictor-65d96b6466-67vtr namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: 0976311d-7f96-496a-8f90-2411fa4978f6 kind: Event lastTimestamp: "2026-07-07T14:31:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:31:01Z" name: raw-sklearn-73688-predictor-65d96b6466-67vtr.18c008601b8f828e namespace: kserve-ci-e2e-test resourceVersion: "20550" uid: c4e13d11-bd50-4700-b8c2-09dfb67bf593 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-73688-predictor-65d96b6466-67vtr namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: 0976311d-7f96-496a-8f90-2411fa4978f6 kind: Event lastTimestamp: "2026-07-07T14:31:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:31:01Z" name: raw-sklearn-73688-predictor-65d96b6466-67vtr.18c008601e5764eb namespace: kserve-ci-e2e-test resourceVersion: "20554" uid: 4790bb8b-0796-4ef9-a947-5ee85ea0f09e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:31:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-73688-predictor-65d96b6466-67vtr namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: 0976311d-7f96-496a-8f90-2411fa4978f6 kind: Event lastTimestamp: "2026-07-07T14:31:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:31:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:31:01Z" name: raw-sklearn-73688-predictor-65d96b6466-67vtr.18c008601ecb144b namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: bc557d5f-8e6b-4c69-9c32-48b912dcca8b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T14:31:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-73688-predictor-65d96b6466-67vtr namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: 0976311d-7f96-496a-8f90-2411fa4978f6 kind: Event lastTimestamp: "2026-07-07T14:32:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:31:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:32:13Z" name: raw-sklearn-73688-predictor-65d96b6466-67vtr.18c008608fc399f6 namespace: kserve-ci-e2e-test resourceVersion: "21069" uid: 8ad66afa-0ac4-4d26-b0b5-54794c510696 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:32:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-73688-predictor-65d96b6466-67vtr namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: 0976311d-7f96-496a-8f90-2411fa4978f6 kind: Event lastTimestamp: "2026-07-07T14:32:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:32:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:32:26Z" name: raw-sklearn-73688-predictor-65d96b6466-67vtr.18c00874094eaaeb namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: b0f7922e-b636-415e-b060-a11e1a1c7588 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:32:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-73688-predictor-65d96b6466-67vtr namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: 0976311d-7f96-496a-8f90-2411fa4978f6 kind: Event lastTimestamp: "2026-07-07T14:32:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:32:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:32:26Z" name: raw-sklearn-73688-predictor-65d96b6466-67vtr.18c00874094f6758 namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: f61341ce-2ef9-4750-a851-0adb27fa218e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:32:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-73688-predictor-65d96b6466-67vtr namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: 0976311d-7f96-496a-8f90-2411fa4978f6 kind: Event lastTimestamp: "2026-07-07T14:32:29Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:32:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:32:29Z" name: raw-sklearn-73688-predictor-65d96b6466-67vtr.18c00874960a765c namespace: kserve-ci-e2e-test resourceVersion: "21208" uid: 4a70b941-bade-4708-8990-57d79275ffcc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-73688-predictor-65d96b6466 namespace: kserve-ci-e2e-test resourceVersion: "20479" uid: 2ff32b89-5896-4aa0-bea5-5cc357d70d93 kind: Event lastTimestamp: "2026-07-07T14:30:56Z" message: 'Created pod: raw-sklearn-73688-predictor-65d96b6466-67vtr' metadata: creationTimestamp: "2026-07-07T14:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:30:56Z" name: raw-sklearn-73688-predictor-65d96b6466.18c0085f0972b84e namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: 58eb9e32-cb90-4992-aa44-e34e8e823a61 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-73688-predictor namespace: kserve-ci-e2e-test resourceVersion: "20477" uid: 8994017a-174e-4265-8762-9372d04e0fcd kind: Event lastTimestamp: "2026-07-07T14:30:56Z" message: Scaled up replica set raw-sklearn-73688-predictor-65d96b6466 from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:30:56Z" name: raw-sklearn-73688-predictor.18c0085f06330344 namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: b6b44f98-311a-42f7-9835-807ee9aefccd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:31:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-73688-predictor namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: 687dfff3-0a77-4f54-b6fd-8a51e2016bf7 kind: Event lastTimestamp: "2026-07-07T14:31:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:31:26Z" name: raw-sklearn-73688-predictor.18c0086285f2d2cc namespace: kserve-ci-e2e-test resourceVersion: "20741" uid: 1eeb7b24-9cdb-4ed3-8ed8-ace4bb44eac8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:31:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-73688-predictor namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: 687dfff3-0a77-4f54-b6fd-8a51e2016bf7 kind: Event lastTimestamp: "2026-07-07T14:31:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:31:26Z" name: raw-sklearn-73688-predictor.18c00862866996e2 namespace: kserve-ci-e2e-test resourceVersion: "20742" uid: 7d19eda2-cf29-423d-ac0b-42a2cf13d94f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:31:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-73688-predictor namespace: kserve-ci-e2e-test resourceVersion: "20634" uid: 687dfff3-0a77-4f54-b6fd-8a51e2016bf7 kind: Event lastTimestamp: "2026-07-07T14:32:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:32:26Z" name: raw-sklearn-73688-predictor.18c00869850c2ad1 namespace: kserve-ci-e2e-test resourceVersion: "21171" uid: 8081050e-610d-4dbf-8a44-20987f5a3eb1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-07T14:31:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-73688-predictor namespace: kserve-ci-e2e-test resourceVersion: "20634" uid: 687dfff3-0a77-4f54-b6fd-8a51e2016bf7 kind: Event lastTimestamp: "2026-07-07T14:32:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:32:26Z" name: raw-sklearn-73688-predictor.18c00869858c1a4e namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: 77d45701-5d05-42bc-847a-ace54592ce4e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-07T14:30:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-73688 namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: fd2e41c3-b191-4318-ac22-652217587373 kind: Event lastTimestamp: "2026-07-07T14:32:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:32:23Z" name: raw-sklearn-73688.18c0085f05ff9c4b namespace: kserve-ci-e2e-test resourceVersion: "21144" uid: 808b862a-3f6c-43f0-ae20-38046899fc56 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:30:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-73688 namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: fd2e41c3-b191-4318-ac22-652217587373 kind: Event lastTimestamp: "2026-07-07T14:30:56Z" message: 'Failed to update status for InferenceService "raw-sklearn-73688": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-73688": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:30:56Z" name: raw-sklearn-73688.18c0085f0656422a namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: fe1c20e4-66b3-47c7-bca2-bf47446b36aa reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:32:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-73688 namespace: kserve-ci-e2e-test resourceVersion: "21142" uid: fd2e41c3-b191-4318-ac22-652217587373 kind: Event lastTimestamp: "2026-07-07T14:32:23Z" message: InferenceService [raw-sklearn-73688] is Ready metadata: creationTimestamp: "2026-07-07T14:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:32:23Z" name: raw-sklearn-73688.18c008732c1939a9 namespace: kserve-ci-e2e-test resourceVersion: "21143" uid: 4c1fabf8-07f3-404e-b88f-06275b0c52ee reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-07T14:32:32.079510Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l namespace: kserve-ci-e2e-test resourceVersion: "21249" uid: a9d6b4ea-71ca-4c11-bb62-79e6c0eb71b9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l to ip-10-0-140-100.ec2.internal metadata: creationTimestamp: "2026-07-07T14:32:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-07T14:32:32Z" name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l.18c008753dc96076 namespace: kserve-ci-e2e-test resourceVersion: "21253" uid: c453d9f5-d64a-497a-b053-beb62cfe8178 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84448d6d9-ddnp2 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:32:32Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: a9d6b4ea-71ca-4c11-bb62-79e6c0eb71b9 kind: Event lastTimestamp: "2026-07-07T14:32:32Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-runtime-b5c4a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-07T14:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:32:32Z" name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l.18c0087545ce9fff namespace: kserve-ci-e2e-test resourceVersion: "21264" uid: 626b9e15-f44c-4660-bf1b-6f2cbd4870d6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:32:33Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l namespace: kserve-ci-e2e-test resourceVersion: "21261" uid: a9d6b4ea-71ca-4c11-bb62-79e6c0eb71b9 kind: Event lastTimestamp: "2026-07-07T14:32:33Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-07T14:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-07T14:32:33Z" name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l.18c008757add892e namespace: kserve-ci-e2e-test resourceVersion: "21278" uid: ea75ae21-9ec4-47ad-a14a-9e1f74b853c9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:32:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: a9d6b4ea-71ca-4c11-bb62-79e6c0eb71b9 kind: Event lastTimestamp: "2026-07-07T14:32:33Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:13ad765c0f323200e1db7b45c99661287e65c5158d6b54b16139829636d20be4" already present on machine metadata: creationTimestamp: "2026-07-07T14:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:32:33Z" name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l.18c008757c6c647f namespace: kserve-ci-e2e-test resourceVersion: "21280" uid: cd944229-fd4f-4253-8e0a-9e43bef71100 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:32:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: a9d6b4ea-71ca-4c11-bb62-79e6c0eb71b9 kind: Event lastTimestamp: "2026-07-07T14:32:33Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-07T14:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:32:33Z" name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l.18c008757f16ea11 namespace: kserve-ci-e2e-test resourceVersion: "21281" uid: 6ef0ae8c-88dc-4cae-a747-b21f8ff963c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:32:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: a9d6b4ea-71ca-4c11-bb62-79e6c0eb71b9 kind: Event lastTimestamp: "2026-07-07T14:32:33Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-07T14:32:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:32:33Z" name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l.18c008757f875af4 namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: afe2ad1d-9188-4ef8-a9c0-d57a03bced65 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: a9d6b4ea-71ca-4c11-bb62-79e6c0eb71b9 kind: Event lastTimestamp: "2026-07-07T14:32:37Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-07T14:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:32:37Z" name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l.18c00876897bf538 namespace: kserve-ci-e2e-test resourceVersion: "21319" uid: 7bca71b3-e965-470a-bfe3-b4eb33de1cac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: a9d6b4ea-71ca-4c11-bb62-79e6c0eb71b9 kind: Event lastTimestamp: "2026-07-07T14:32:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-07T14:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:32:37Z" name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l.18c008768c53138b namespace: kserve-ci-e2e-test resourceVersion: "21321" uid: 64dddd98-c9e0-44c6-909a-111345434fdf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: a9d6b4ea-71ca-4c11-bb62-79e6c0eb71b9 kind: Event lastTimestamp: "2026-07-07T14:32:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-07T14:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:32:37Z" name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l.18c008768cc18941 namespace: kserve-ci-e2e-test resourceVersion: "21322" uid: 87ef7649-b7c5-40fd-b8fd-92e0edeafcb1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: a9d6b4ea-71ca-4c11-bb62-79e6c0eb71b9 kind: Event lastTimestamp: "2026-07-07T14:32:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-07T14:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:32:37Z" name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l.18c008768ccbfec1 namespace: kserve-ci-e2e-test resourceVersion: "21323" uid: a7480cf7-559a-4d23-acf6-deead7292c8a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: a9d6b4ea-71ca-4c11-bb62-79e6c0eb71b9 kind: Event lastTimestamp: "2026-07-07T14:32:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-07T14:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:32:37Z" name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l.18c008768f9f6b99 namespace: kserve-ci-e2e-test resourceVersion: "21327" uid: caa74986-c413-4911-873e-b75e501d3131 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:32:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: a9d6b4ea-71ca-4c11-bb62-79e6c0eb71b9 kind: Event lastTimestamp: "2026-07-07T14:32:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:32:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:32:37Z" name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l.18c00876900d88ad namespace: kserve-ci-e2e-test resourceVersion: "21328" uid: 03fb7923-555e-4894-ac99-4a10b9b2dde5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-07T14:32:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: a9d6b4ea-71ca-4c11-bb62-79e6c0eb71b9 kind: Event lastTimestamp: "2026-07-07T14:34:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:32:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:04Z" name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l.18c00876c58477cf namespace: kserve-ci-e2e-test resourceVersion: "21971" uid: ed849272-6c17-487d-95b4-42c253557b8b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: a9d6b4ea-71ca-4c11-bb62-79e6c0eb71b9 kind: Event lastTimestamp: "2026-07-07T14:34:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-07T14:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:02Z" name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l.18c0088a35e560f7 namespace: kserve-ci-e2e-test resourceVersion: "21949" uid: f3302cb4-8d14-4e3e-84b7-1d9cd80d5944 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: a9d6b4ea-71ca-4c11-bb62-79e6c0eb71b9 kind: Event lastTimestamp: "2026-07-07T14:34:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-07T14:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:02Z" name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l.18c0088a35e5e62c namespace: kserve-ci-e2e-test resourceVersion: "21950" uid: fa9a09ac-2e7d-4238-b409-cbb7e09a8842 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:34:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: a9d6b4ea-71ca-4c11-bb62-79e6c0eb71b9 kind: Event lastTimestamp: "2026-07-07T14:34:04Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-07T14:34:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-07T14:34:04Z" name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l.18c0088acbc4e62f namespace: kserve-ci-e2e-test resourceVersion: "21970" uid: a3415269-bde4-4305-8756-9bbf1e031986 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-100.ec2.internal source: component: kubelet host: ip-10-0-140-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:32:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd namespace: kserve-ci-e2e-test resourceVersion: "21238" uid: 7ac256db-ab0e-4c0a-8998-ff992a577f21 kind: Event lastTimestamp: "2026-07-07T14:32:32Z" message: 'Created pod: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd-wqj8l' metadata: creationTimestamp: "2026-07-07T14:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:32:32Z" name: raw-sklearn-runtime-b5c4a-predictor-9cf4568fd.18c008753d31db18 namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 9083a54d-2730-4c9e-ad7d-2f0a69efcd5e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:32:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-b5c4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21236" uid: 25004c83-d514-42f0-81db-b58aec133f3e kind: Event lastTimestamp: "2026-07-07T14:32:31Z" message: Scaled up replica set raw-sklearn-runtime-b5c4a-predictor-9cf4568fd from 0 to 1 metadata: creationTimestamp: "2026-07-07T14:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:32:31Z" name: raw-sklearn-runtime-b5c4a-predictor.18c00875384d488a namespace: kserve-ci-e2e-test resourceVersion: "21240" uid: c3e68455-0a51-4332-9eb9-4eaf97acf88e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:32:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-b5c4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21237" uid: a84593aa-3c6d-47c8-b31f-d9894d4d2bad kind: Event lastTimestamp: "2026-07-07T14:33:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:33:17Z" name: raw-sklearn-runtime-b5c4a-predictor.18c00878b7fe8be5 namespace: kserve-ci-e2e-test resourceVersion: "21609" uid: 7caf9708-ca9e-478b-965e-f63a1e2dba5f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:32:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-b5c4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21237" uid: a84593aa-3c6d-47c8-b31f-d9894d4d2bad kind: Event lastTimestamp: "2026-07-07T14:33:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-07T14:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:33:17Z" name: raw-sklearn-runtime-b5c4a-predictor.18c00878b8d7e895 namespace: kserve-ci-e2e-test resourceVersion: "21611" uid: d2c9fba0-8081-4866-a344-3807772926e0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:33:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-b5c4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: a84593aa-3c6d-47c8-b31f-d9894d4d2bad kind: Event lastTimestamp: "2026-07-07T14:33:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:33:47Z" name: raw-sklearn-runtime-b5c4a-predictor.18c0087c3871c105 namespace: kserve-ci-e2e-test resourceVersion: "21819" uid: 73e80422-0758-4d99-bf01-49faf322d47a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-07T14:33:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-b5c4a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: a84593aa-3c6d-47c8-b31f-d9894d4d2bad kind: Event lastTimestamp: "2026-07-07T14:33:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-07T14:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-07T14:33:47Z" name: raw-sklearn-runtime-b5c4a-predictor.18c0087c393bfc91 namespace: kserve-ci-e2e-test resourceVersion: "21820" uid: e7eddf1d-3b68-4382-b58c-d949c6ca8053 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-07T14:32:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-b5c4a namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: 538b17a8-7200-4c2a-806a-edf720d4e5e4 kind: Event lastTimestamp: "2026-07-07T14:33:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-07T14:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:33:54Z" name: raw-sklearn-runtime-b5c4a.18c0087538148281 namespace: kserve-ci-e2e-test resourceVersion: "21880" uid: 2ef7c4f2-6a91-4e74-90c5-f37997b6f014 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-07T14:32:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-b5c4a namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: 538b17a8-7200-4c2a-806a-edf720d4e5e4 kind: Event lastTimestamp: "2026-07-07T14:32:32Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-b5c4a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-b5c4a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:32:32Z" name: raw-sklearn-runtime-b5c4a.18c0087538d39eea namespace: kserve-ci-e2e-test resourceVersion: "21258" uid: 817f2c2b-c60e-4a51-af37-14e7cae623e8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:32:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-b5c4a namespace: kserve-ci-e2e-test resourceVersion: "21235" uid: 538b17a8-7200-4c2a-806a-edf720d4e5e4 kind: Event lastTimestamp: "2026-07-07T14:32:32Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-b5c4a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-07T14:32:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:32:32Z" name: raw-sklearn-runtime-b5c4a.18c008753f84cc18 namespace: kserve-ci-e2e-test resourceVersion: "21259" uid: a1694563-cbc2-4c40-8d97-55568b074ca6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-07T14:33:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-b5c4a namespace: kserve-ci-e2e-test resourceVersion: "21877" uid: 538b17a8-7200-4c2a-806a-edf720d4e5e4 kind: Event lastTimestamp: "2026-07-07T14:33:54Z" message: InferenceService [raw-sklearn-runtime-b5c4a] is Ready metadata: creationTimestamp: "2026-07-07T14:33:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-07T14:33:54Z" name: raw-sklearn-runtime-b5c4a.18c008887e707ce8 namespace: kserve-ci-e2e-test resourceVersion: "21878" uid: 12d1662a-63c0-4577-bee9-a513541f77d6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "22213"