--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:59:23.019278Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-88a6a6-predictor-67f959bc6b-4g4hr namespace: kserve-ci-e2e-test resourceVersion: "20109" uid: e20f84f2-25b2-47bf-a495-092acea6c5db kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-init-fail-88a6a6-predictor-67f959bc6b-4g4hr to ip-10-0-134-20.ec2.internal metadata: creationTimestamp: "2026-07-01T14:59:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:59:23Z" name: isvc-init-fail-88a6a6-predictor-67f959bc6b-4g4hr.18be3270e96d7f9e namespace: kserve-ci-e2e-test resourceVersion: "20116" uid: c45a2faa-7b3e-4230-bb63-b3eabaa255b1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-655cd54c75-lthr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:23Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-88a6a6-predictor-67f959bc6b-4g4hr namespace: kserve-ci-e2e-test resourceVersion: "20114" uid: e20f84f2-25b2-47bf-a495-092acea6c5db kind: Event lastTimestamp: "2026-07-01T14:59:23Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-init-fail-88a6a6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T14:59:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:23Z" name: isvc-init-fail-88a6a6-predictor-67f959bc6b-4g4hr.18be3270f36dffc8 namespace: kserve-ci-e2e-test resourceVersion: "20123" uid: e2f79f30-db37-4544-92f5-e218a95b01f7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:24Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-init-fail-88a6a6-predictor-67f959bc6b-4g4hr namespace: kserve-ci-e2e-test resourceVersion: "20120" uid: e20f84f2-25b2-47bf-a495-092acea6c5db kind: Event lastTimestamp: "2026-07-01T14:59:24Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:59:24Z" name: isvc-init-fail-88a6a6-predictor-67f959bc6b-4g4hr.18be3271267d94e6 namespace: kserve-ci-e2e-test resourceVersion: "20131" uid: 92a51fb0-6dfa-4cb5-947a-e17e75d4e072 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:59:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-88a6a6-predictor-67f959bc6b-4g4hr namespace: kserve-ci-e2e-test resourceVersion: "20114" uid: e20f84f2-25b2-47bf-a495-092acea6c5db kind: Event lastTimestamp: "2026-07-01T14:59:27Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:32825628ffab44a0db6a86e1f74488e47dcab508ff646215bea8b81168e85657" already present on machine metadata: creationTimestamp: "2026-07-01T14:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:27Z" name: isvc-init-fail-88a6a6-predictor-67f959bc6b-4g4hr.18be3271281d75e6 namespace: kserve-ci-e2e-test resourceVersion: "20167" uid: 03a8c135-a7a7-4b57-8cfe-771ee5ba2aa9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:59:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-88a6a6-predictor-67f959bc6b-4g4hr namespace: kserve-ci-e2e-test resourceVersion: "20114" uid: e20f84f2-25b2-47bf-a495-092acea6c5db kind: Event lastTimestamp: "2026-07-01T14:59:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T14:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:27Z" name: isvc-init-fail-88a6a6-predictor-67f959bc6b-4g4hr.18be32712ad14f84 namespace: kserve-ci-e2e-test resourceVersion: "20169" uid: 7a9784ff-6933-4c08-8a60-42a826cbea1e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:59:24Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-88a6a6-predictor-67f959bc6b-4g4hr namespace: kserve-ci-e2e-test resourceVersion: "20114" uid: e20f84f2-25b2-47bf-a495-092acea6c5db kind: Event lastTimestamp: "2026-07-01T14:59:27Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T14:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:27Z" name: isvc-init-fail-88a6a6-predictor-67f959bc6b-4g4hr.18be32712b3e020d namespace: kserve-ci-e2e-test resourceVersion: "20171" uid: 72164b12-46f3-479e-a175-089867b73830 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-init-fail-88a6a6-predictor-67f959bc6b-4g4hr namespace: kserve-ci-e2e-test resourceVersion: "20114" uid: e20f84f2-25b2-47bf-a495-092acea6c5db kind: Event lastTimestamp: "2026-07-01T14:59:28Z" message: Stopping container storage-initializer metadata: creationTimestamp: "2026-07-01T14:59:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:28Z" name: isvc-init-fail-88a6a6-predictor-67f959bc6b-4g4hr.18be327237862081 namespace: kserve-ci-e2e-test resourceVersion: "20197" uid: 22879e53-36ed-40e4-b6b7-72636203b54a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-init-fail-88a6a6-predictor-67f959bc6b namespace: kserve-ci-e2e-test resourceVersion: "20102" uid: 40250015-77c6-4abd-a20f-2a74d6d66607 kind: Event lastTimestamp: "2026-07-01T14:59:23Z" message: 'Created pod: isvc-init-fail-88a6a6-predictor-67f959bc6b-4g4hr' metadata: creationTimestamp: "2026-07-01T14:59:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:59:23Z" name: isvc-init-fail-88a6a6-predictor-67f959bc6b.18be3270e8c3c162 namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: a733bbc8-4ecb-46d5-9ab7-6f81d024174f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-init-fail-88a6a6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20100" uid: 649a26a3-a184-438b-80a7-afb820d3e316 kind: Event lastTimestamp: "2026-07-01T14:59:22Z" message: Scaled up replica set isvc-init-fail-88a6a6-predictor-67f959bc6b from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:59:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:59:22Z" name: isvc-init-fail-88a6a6-predictor.18be3270e545c103 namespace: kserve-ci-e2e-test resourceVersion: "20103" uid: b026a0d6-fa8b-478b-8f30-2a7ba1390490 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T14:59:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-88a6a6 namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: 6b6ea9c3-9fb7-4c46-81ee-05b8c63dfe72 kind: Event lastTimestamp: "2026-07-01T14:59:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:59:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:59:27Z" name: isvc-init-fail-88a6a6.18be3270e5488c55 namespace: kserve-ci-e2e-test resourceVersion: "20174" uid: 17f7bb1c-c349-4851-9e2c-ad1b9e9ac062 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-88a6a6 namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: 6b6ea9c3-9fb7-4c46-81ee-05b8c63dfe72 kind: Event lastTimestamp: "2026-07-01T14:59:22Z" message: 'Failed to update status for InferenceService "isvc-init-fail-88a6a6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-88a6a6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:59:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:59:22Z" name: isvc-init-fail-88a6a6.18be3270e5a352fd namespace: kserve-ci-e2e-test resourceVersion: "20106" uid: 7d697d2c-49bc-4552-8767-4286be4b1a83 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-init-fail-88a6a6 namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: 6b6ea9c3-9fb7-4c46-81ee-05b8c63dfe72 kind: Event lastTimestamp: "2026-07-01T14:59:22Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-init-fail-88a6a6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:59:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:59:22Z" name: isvc-init-fail-88a6a6.18be3270e5a3ab3a namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 2356ba08-1bd4-4f46-93ec-b1ac4a1531ae reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:53:10.985596Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 namespace: kserve-ci-e2e-test resourceVersion: "17091" uid: 5b680726-af0f-4165-9c8b-0f59dd0d42a6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 to ip-10-0-134-20.ec2.internal metadata: creationTimestamp: "2026-07-01T14:53:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:53:10Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8.18be321a4a7df8f5 namespace: kserve-ci-e2e-test resourceVersion: "17096" uid: fe36f947-e7d2-43ab-9f5b-a74f51423b13 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-655cd54c75-lthr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:53:11Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 namespace: kserve-ci-e2e-test resourceVersion: "17100" uid: 5b680726-af0f-4165-9c8b-0f59dd0d42a6 kind: Event lastTimestamp: "2026-07-01T14:53:11Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:53:11Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8.18be321a63ecdcdb namespace: kserve-ci-e2e-test resourceVersion: "17107" uid: 5f26737c-4af8-44b2-a249-3ae18d299282 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 5b680726-af0f-4165-9c8b-0f59dd0d42a6 kind: Event lastTimestamp: "2026-07-01T14:53:11Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:32825628ffab44a0db6a86e1f74488e47dcab508ff646215bea8b81168e85657" already present on machine metadata: creationTimestamp: "2026-07-01T14:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:53:11Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8.18be321a6640ddb1 namespace: kserve-ci-e2e-test resourceVersion: "17109" uid: 8917194f-267f-45f7-8db8-8ebb2f1b48eb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 5b680726-af0f-4165-9c8b-0f59dd0d42a6 kind: Event lastTimestamp: "2026-07-01T14:53:11Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T14:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:53:11Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8.18be321a691576e3 namespace: kserve-ci-e2e-test resourceVersion: "17112" uid: 2528971a-05c7-4c59-9490-b56f06917d90 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:53:11Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 5b680726-af0f-4165-9c8b-0f59dd0d42a6 kind: Event lastTimestamp: "2026-07-01T14:53:11Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T14:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:53:11Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8.18be321a698cba38 namespace: kserve-ci-e2e-test resourceVersion: "17114" uid: 1a1bb244-783c-4091-8d80-721ab8940921 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:53:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 5b680726-af0f-4165-9c8b-0f59dd0d42a6 kind: Event lastTimestamp: "2026-07-01T14:53:16Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T14:53:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:53:16Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8.18be321b763eada0 namespace: kserve-ci-e2e-test resourceVersion: "17151" uid: 9880122a-22f7-4ee9-8ee2-1e809a5e797e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:53:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 5b680726-af0f-4165-9c8b-0f59dd0d42a6 kind: Event lastTimestamp: "2026-07-01T14:53:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T14:53:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:53:16Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8.18be321b79736be9 namespace: kserve-ci-e2e-test resourceVersion: "17153" uid: 9b1762fa-feee-4990-9d71-e28172d08546 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:53:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 5b680726-af0f-4165-9c8b-0f59dd0d42a6 kind: Event lastTimestamp: "2026-07-01T14:53:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T14:53:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:53:16Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8.18be321b79e361f9 namespace: kserve-ci-e2e-test resourceVersion: "17154" uid: 778eab91-f798-49cf-8f6e-d27717c66a29 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:53:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 5b680726-af0f-4165-9c8b-0f59dd0d42a6 kind: Event lastTimestamp: "2026-07-01T14:53:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T14:53:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:53:16Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8.18be321b79ed34cb namespace: kserve-ci-e2e-test resourceVersion: "17155" uid: 4d524317-059d-4b11-9129-11e412d0cf40 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:53:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 5b680726-af0f-4165-9c8b-0f59dd0d42a6 kind: Event lastTimestamp: "2026-07-01T14:53:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T14:53:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:53:16Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8.18be321b7cca6070 namespace: kserve-ci-e2e-test resourceVersion: "17159" uid: 6aaa40b9-15d1-4196-a0d3-2996136a073d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:53:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 5b680726-af0f-4165-9c8b-0f59dd0d42a6 kind: Event lastTimestamp: "2026-07-01T14:53:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:53:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:53:16Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8.18be321b7d361c16 namespace: kserve-ci-e2e-test resourceVersion: "17160" uid: c6558508-9c2f-40a8-bb9f-385dcbe75052 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:53:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 5b680726-af0f-4165-9c8b-0f59dd0d42a6 kind: Event lastTimestamp: "2026-07-01T14:53:16Z" message: Container image "quay.io/opendatahub/kserve-agent@sha256:b968ed73a63cc06f7467120ce75d205a47a53bf36fad1b5d010a25df8b9285c9" already present on machine metadata: creationTimestamp: "2026-07-01T14:53:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:53:16Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8.18be321b7d3ff3ec namespace: kserve-ci-e2e-test resourceVersion: "17161" uid: b916ef23-68b1-4cd0-b076-51ec8b8b89a1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:53:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 5b680726-af0f-4165-9c8b-0f59dd0d42a6 kind: Event lastTimestamp: "2026-07-01T14:53:16Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-01T14:53:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:53:16Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8.18be321b7fe8ab2e namespace: kserve-ci-e2e-test resourceVersion: "17162" uid: ef7d40be-650b-4352-86c2-9f2a303cdea5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:53:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 5b680726-af0f-4165-9c8b-0f59dd0d42a6 kind: Event lastTimestamp: "2026-07-01T14:53:16Z" message: Started container agent metadata: creationTimestamp: "2026-07-01T14:53:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:53:16Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8.18be321b80538f05 namespace: kserve-ci-e2e-test resourceVersion: "17163" uid: e52b3f05-ae81-42e6-a9bd-95eefa0f9294 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T14:53:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 5b680726-af0f-4165-9c8b-0f59dd0d42a6 kind: Event lastTimestamp: "2026-07-01T14:54:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:53:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:54:54Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8.18be321bb24d4a65 namespace: kserve-ci-e2e-test resourceVersion: "17985" uid: a845e192-aaf0-4818-8904-755413827960 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T14:53:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 5b680726-af0f-4165-9c8b-0f59dd0d42a6 kind: Event lastTimestamp: "2026-07-01T14:54:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T14:53:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:54:54Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8.18be321bee2f3cc4 namespace: kserve-ci-e2e-test resourceVersion: "17986" uid: f0d23793-b701-4097-816c-dd8df405f2b3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:54:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 5b680726-af0f-4165-9c8b-0f59dd0d42a6 kind: Event lastTimestamp: "2026-07-01T14:54:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T14:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:54:46Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8.18be32307bf38908 namespace: kserve-ci-e2e-test resourceVersion: "17866" uid: ba123e69-8b26-4470-9b92-8cd88d3d2f51 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:54:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 5b680726-af0f-4165-9c8b-0f59dd0d42a6 kind: Event lastTimestamp: "2026-07-01T14:54:46Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-01T14:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:54:46Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8.18be32307bf452ea namespace: kserve-ci-e2e-test resourceVersion: "17871" uid: a3f1447e-4efc-45fb-900a-64b9654f555e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:54:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 5b680726-af0f-4165-9c8b-0f59dd0d42a6 kind: Event lastTimestamp: "2026-07-01T14:54:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:54:46Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8.18be32307bf47753 namespace: kserve-ci-e2e-test resourceVersion: "17873" uid: 1f21548e-a107-4658-919e-a1ec80f5ebfe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T14:54:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 5b680726-af0f-4165-9c8b-0f59dd0d42a6 kind: Event lastTimestamp: "2026-07-01T14:55:04Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:54:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:55:04Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8.18be32311e68e450 namespace: kserve-ci-e2e-test resourceVersion: "18060" uid: dd21a788-7111-40b4-b47d-04aeccb06281 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:53:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-logger-raw-4ce12-predictor-567557fdcb namespace: kserve-ci-e2e-test resourceVersion: "17078" uid: 3f98ff1e-39fa-41f8-b1d0-5e9a23227b34 kind: Event lastTimestamp: "2026-07-01T14:53:10Z" message: 'Created pod: isvc-logger-raw-4ce12-predictor-567557fdcb-fvjj8' metadata: creationTimestamp: "2026-07-01T14:53:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:53:10Z" name: isvc-logger-raw-4ce12-predictor-567557fdcb.18be321a49ed1cac namespace: kserve-ci-e2e-test resourceVersion: "17093" uid: 398c5fb4-6c0a-4758-a006-bc536ff1a580 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:53:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-logger-raw-4ce12-predictor namespace: kserve-ci-e2e-test resourceVersion: "17076" uid: f705caf2-a760-4788-818e-24b103369249 kind: Event lastTimestamp: "2026-07-01T14:53:10Z" message: Scaled up replica set isvc-logger-raw-4ce12-predictor-567557fdcb from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:53:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:53:10Z" name: isvc-logger-raw-4ce12-predictor.18be321a456444aa namespace: kserve-ci-e2e-test resourceVersion: "17080" uid: 59d13dd1-f2fa-45bd-ab18-32ec9ea80acc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:53:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-4ce12-predictor namespace: kserve-ci-e2e-test resourceVersion: "17077" uid: ed29f403-6aa6-4f1a-8371-045b93f33879 kind: Event lastTimestamp: "2026-07-01T14:53:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:53:40Z" name: isvc-logger-raw-4ce12-predictor.18be321dc48324f1 namespace: kserve-ci-e2e-test resourceVersion: "17355" uid: 7aa3d7df-ebe9-4f0d-935d-1012f1b63b0f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:53:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-4ce12-predictor namespace: kserve-ci-e2e-test resourceVersion: "17077" uid: ed29f403-6aa6-4f1a-8371-045b93f33879 kind: Event lastTimestamp: "2026-07-01T14:53:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:53:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:53:40Z" name: isvc-logger-raw-4ce12-predictor.18be321dc501822f namespace: kserve-ci-e2e-test resourceVersion: "17356" uid: 011aa946-20b9-4f84-9079-97a03ca16113 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T14:53:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-4ce12-predictor namespace: kserve-ci-e2e-test resourceVersion: "17248" uid: ed29f403-6aa6-4f1a-8371-045b93f33879 kind: Event lastTimestamp: "2026-07-01T14:54:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:54:40Z" name: isvc-logger-raw-4ce12-predictor.18be3224c2a7a1aa namespace: kserve-ci-e2e-test resourceVersion: "17785" uid: 885693aa-ea87-45b6-949c-0ce575746e50 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T14:53:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-logger-raw-4ce12-predictor namespace: kserve-ci-e2e-test resourceVersion: "17248" uid: ed29f403-6aa6-4f1a-8371-045b93f33879 kind: Event lastTimestamp: "2026-07-01T14:54:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:53:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:54:40Z" name: isvc-logger-raw-4ce12-predictor.18be3224c33c052e namespace: kserve-ci-e2e-test resourceVersion: "17786" uid: 10a5a705-9a8f-4e24-bc1e-1f24ea0a0d2c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T14:53:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-4ce12 namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: 0eba3a7e-eceb-4370-a1b5-c0b9fe58778d kind: Event lastTimestamp: "2026-07-01T14:54:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:53:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:54:34Z" name: isvc-logger-raw-4ce12.18be321a45628a4f namespace: kserve-ci-e2e-test resourceVersion: "17745" uid: f32b0d98-28a5-4468-91d2-76bfeb2c7482 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:53:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-4ce12 namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: 0eba3a7e-eceb-4370-a1b5-c0b9fe58778d kind: Event lastTimestamp: "2026-07-01T14:53:10Z" message: 'Failed to update status for InferenceService "isvc-logger-raw-4ce12": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-logger-raw-4ce12": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:53:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:53:10Z" name: isvc-logger-raw-4ce12.18be321a45d4e200 namespace: kserve-ci-e2e-test resourceVersion: "17083" uid: 68c887df-821f-4783-8790-0187db97fc31 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:54:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-logger-raw-4ce12 namespace: kserve-ci-e2e-test resourceVersion: "17743" uid: 0eba3a7e-eceb-4370-a1b5-c0b9fe58778d kind: Event lastTimestamp: "2026-07-01T14:54:34Z" message: InferenceService [isvc-logger-raw-4ce12] is Ready metadata: creationTimestamp: "2026-07-01T14:54:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:54:34Z" name: isvc-logger-raw-4ce12.18be322da7cd1442 namespace: kserve-ci-e2e-test resourceVersion: "17744" uid: 006d1719-f8df-4946-b9a9-1e3d404acc09 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:57:31.555227Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-df069b-predictor-8597f6d579-wrvrh namespace: kserve-ci-e2e-test resourceVersion: "19147" uid: c5a2cb72-1399-4e30-83aa-ccd8c293828d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-primary-df069b-predictor-8597f6d579-wrvrh to ip-10-0-134-20.ec2.internal metadata: creationTimestamp: "2026-07-01T14:57:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:57:31Z" name: isvc-primary-df069b-predictor-8597f6d579-wrvrh.18be3256f5a70fb1 namespace: kserve-ci-e2e-test resourceVersion: "19151" uid: 4585a1b9-bfa7-4289-9e10-14330dbe8ef1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-655cd54c75-lthr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:57:31Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-primary-df069b-predictor-8597f6d579-wrvrh namespace: kserve-ci-e2e-test resourceVersion: "19157" uid: c5a2cb72-1399-4e30-83aa-ccd8c293828d kind: Event lastTimestamp: "2026-07-01T14:57:31Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:57:31Z" name: isvc-primary-df069b-predictor-8597f6d579-wrvrh.18be32570f29a0a5 namespace: kserve-ci-e2e-test resourceVersion: "19167" uid: ebbc2bc4-b4db-44f7-868c-9298a73afa08 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:57:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-df069b-predictor-8597f6d579-wrvrh namespace: kserve-ci-e2e-test resourceVersion: "19149" uid: c5a2cb72-1399-4e30-83aa-ccd8c293828d kind: Event lastTimestamp: "2026-07-01T14:57:32Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:32825628ffab44a0db6a86e1f74488e47dcab508ff646215bea8b81168e85657" already present on machine metadata: creationTimestamp: "2026-07-01T14:57:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:57:32Z" name: isvc-primary-df069b-predictor-8597f6d579-wrvrh.18be3257116b63f3 namespace: kserve-ci-e2e-test resourceVersion: "19169" uid: 77f5461c-1e7d-46c2-a3c9-6eb22239f846 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:57:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-df069b-predictor-8597f6d579-wrvrh namespace: kserve-ci-e2e-test resourceVersion: "19149" uid: c5a2cb72-1399-4e30-83aa-ccd8c293828d kind: Event lastTimestamp: "2026-07-01T14:57:32Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T14:57:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:57:32Z" name: isvc-primary-df069b-predictor-8597f6d579-wrvrh.18be32571459ec90 namespace: kserve-ci-e2e-test resourceVersion: "19170" uid: bc6ad5fe-0b3a-419f-9c2c-21565342d6a6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:57:32Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-primary-df069b-predictor-8597f6d579-wrvrh namespace: kserve-ci-e2e-test resourceVersion: "19149" uid: c5a2cb72-1399-4e30-83aa-ccd8c293828d kind: Event lastTimestamp: "2026-07-01T14:57:32Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T14:57:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:57:32Z" name: isvc-primary-df069b-predictor-8597f6d579-wrvrh.18be325714c6dd83 namespace: kserve-ci-e2e-test resourceVersion: "19171" uid: de1f7b52-614e-49aa-897d-c6abe1ac71fe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:57:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-df069b-predictor-8597f6d579-wrvrh namespace: kserve-ci-e2e-test resourceVersion: "19149" uid: c5a2cb72-1399-4e30-83aa-ccd8c293828d kind: Event lastTimestamp: "2026-07-01T14:57:36Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T14:57:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:57:36Z" name: isvc-primary-df069b-predictor-8597f6d579-wrvrh.18be32580682e5b6 namespace: kserve-ci-e2e-test resourceVersion: "19207" uid: 7eb593de-5d91-414c-ad11-e53074a56153 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:57:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-df069b-predictor-8597f6d579-wrvrh namespace: kserve-ci-e2e-test resourceVersion: "19149" uid: c5a2cb72-1399-4e30-83aa-ccd8c293828d kind: Event lastTimestamp: "2026-07-01T14:57:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T14:57:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:57:36Z" name: isvc-primary-df069b-predictor-8597f6d579-wrvrh.18be32580bc0d01c namespace: kserve-ci-e2e-test resourceVersion: "19211" uid: 154f3e0c-90cc-4aae-9e3d-bb258053743a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:57:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-df069b-predictor-8597f6d579-wrvrh namespace: kserve-ci-e2e-test resourceVersion: "19149" uid: c5a2cb72-1399-4e30-83aa-ccd8c293828d kind: Event lastTimestamp: "2026-07-01T14:57:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T14:57:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:57:36Z" name: isvc-primary-df069b-predictor-8597f6d579-wrvrh.18be32580c5df40d namespace: kserve-ci-e2e-test resourceVersion: "19212" uid: 353bbd30-29a2-49a3-b349-9a89b588e433 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:57:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-df069b-predictor-8597f6d579-wrvrh namespace: kserve-ci-e2e-test resourceVersion: "19149" uid: c5a2cb72-1399-4e30-83aa-ccd8c293828d kind: Event lastTimestamp: "2026-07-01T14:57:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T14:57:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:57:36Z" name: isvc-primary-df069b-predictor-8597f6d579-wrvrh.18be32580c6b61ce namespace: kserve-ci-e2e-test resourceVersion: "19213" uid: bc09a890-43e1-465a-94b6-27bd99a79a53 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:57:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-df069b-predictor-8597f6d579-wrvrh namespace: kserve-ci-e2e-test resourceVersion: "19149" uid: c5a2cb72-1399-4e30-83aa-ccd8c293828d kind: Event lastTimestamp: "2026-07-01T14:57:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T14:57:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:57:36Z" name: isvc-primary-df069b-predictor-8597f6d579-wrvrh.18be3258104784a8 namespace: kserve-ci-e2e-test resourceVersion: "19215" uid: 4d71ba71-0d9b-4e3f-a782-3907bf24ce49 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:57:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-df069b-predictor-8597f6d579-wrvrh namespace: kserve-ci-e2e-test resourceVersion: "19149" uid: c5a2cb72-1399-4e30-83aa-ccd8c293828d kind: Event lastTimestamp: "2026-07-01T14:57:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:57:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:57:36Z" name: isvc-primary-df069b-predictor-8597f6d579-wrvrh.18be325810c9b881 namespace: kserve-ci-e2e-test resourceVersion: "19216" uid: bce72fc6-9462-455c-b418-7b3812978e46 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T14:57:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-df069b-predictor-8597f6d579-wrvrh namespace: kserve-ci-e2e-test resourceVersion: "19149" uid: c5a2cb72-1399-4e30-83aa-ccd8c293828d kind: Event lastTimestamp: "2026-07-01T14:58:43Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:57:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:58:43Z" name: isvc-primary-df069b-predictor-8597f6d579-wrvrh.18be325842afc485 namespace: kserve-ci-e2e-test resourceVersion: "19687" uid: 1294901d-0d3b-4df6-877d-85958e087d4c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-primary-df069b-predictor-8597f6d579-wrvrh namespace: kserve-ci-e2e-test resourceVersion: "19149" uid: c5a2cb72-1399-4e30-83aa-ccd8c293828d kind: Event lastTimestamp: "2026-07-01T14:59:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T14:59:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:17Z" name: isvc-primary-df069b-predictor-8597f6d579-wrvrh.18be326fb7afcec8 namespace: kserve-ci-e2e-test resourceVersion: "20043" uid: 893c0ba6-ed47-4b16-b052-447a266f6057 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-df069b-predictor-8597f6d579-wrvrh namespace: kserve-ci-e2e-test resourceVersion: "19149" uid: c5a2cb72-1399-4e30-83aa-ccd8c293828d kind: Event lastTimestamp: "2026-07-01T14:59:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:59:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:17Z" name: isvc-primary-df069b-predictor-8597f6d579-wrvrh.18be326fb7b0816b namespace: kserve-ci-e2e-test resourceVersion: "20044" uid: b6d28a44-8a13-4505-b2b4-69ba19673cc3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-primary-df069b-predictor-8597f6d579-wrvrh namespace: kserve-ci-e2e-test resourceVersion: "19149" uid: c5a2cb72-1399-4e30-83aa-ccd8c293828d kind: Event lastTimestamp: "2026-07-01T14:59:18Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:59:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:18Z" name: isvc-primary-df069b-predictor-8597f6d579-wrvrh.18be326fc6f677be namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: 4d7eb0fd-8b34-4fe5-883a-231e6bd3bc30 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:57:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-primary-df069b-predictor-8597f6d579 namespace: kserve-ci-e2e-test resourceVersion: "19138" uid: 200f1a87-e542-4b1e-9078-0658b643542b kind: Event lastTimestamp: "2026-07-01T14:57:31Z" message: 'Created pod: isvc-primary-df069b-predictor-8597f6d579-wrvrh' metadata: creationTimestamp: "2026-07-01T14:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:57:31Z" name: isvc-primary-df069b-predictor-8597f6d579.18be3256f507ab97 namespace: kserve-ci-e2e-test resourceVersion: "19148" uid: fa2e494d-8267-4611-951f-aeba4c528c95 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:57:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-primary-df069b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19136" uid: 9c0baf2f-c321-4a09-9be2-145d1287b758 kind: Event lastTimestamp: "2026-07-01T14:57:31Z" message: Scaled up replica set isvc-primary-df069b-predictor-8597f6d579 from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:57:31Z" name: isvc-primary-df069b-predictor.18be3256f03380a1 namespace: kserve-ci-e2e-test resourceVersion: "19140" uid: caa03c8e-b663-4511-afac-646eb2e58b57 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:57:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-df069b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19137" uid: 854e0b32-9c65-49f3-8fbd-d17839c2f6b3 kind: Event lastTimestamp: "2026-07-01T14:57:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:57:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:57:46Z" name: isvc-primary-df069b-predictor.18be325a6ef3ad56 namespace: kserve-ci-e2e-test resourceVersion: "19292" uid: f64017c5-1a25-4350-bc3c-1eec4361c427 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:57:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-df069b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19137" uid: 854e0b32-9c65-49f3-8fbd-d17839c2f6b3 kind: Event lastTimestamp: "2026-07-01T14:57:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:57:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:57:46Z" name: isvc-primary-df069b-predictor.18be325a6f6d8bce namespace: kserve-ci-e2e-test resourceVersion: "19294" uid: bb85090e-4335-472a-9eb5-52bb49724004 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T14:58:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-df069b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19293" uid: 854e0b32-9c65-49f3-8fbd-d17839c2f6b3 kind: Event lastTimestamp: "2026-07-01T14:59:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:58:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:59:16Z" name: isvc-primary-df069b-predictor.18be325dee9a8940 namespace: kserve-ci-e2e-test resourceVersion: "19996" uid: d19621ff-0c04-451f-9dde-22b7a6ea9fab reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T14:58:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-primary-df069b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19293" uid: 854e0b32-9c65-49f3-8fbd-d17839c2f6b3 kind: Event lastTimestamp: "2026-07-01T14:59:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:58:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:59:16Z" name: isvc-primary-df069b-predictor.18be325def11228c namespace: kserve-ci-e2e-test resourceVersion: "19997" uid: 848b30d7-22cd-4fad-971f-dbf5395d2b7f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T14:57:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-df069b namespace: kserve-ci-e2e-test resourceVersion: "19127" uid: d06a0a22-f1fb-4622-9466-a849a3042a7f kind: Event lastTimestamp: "2026-07-01T14:58:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:58:53Z" name: isvc-primary-df069b.18be3256f0356dfc namespace: kserve-ci-e2e-test resourceVersion: "19770" uid: 0abe79df-ec52-4d70-9b86-2768c56b463c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:57:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-df069b namespace: kserve-ci-e2e-test resourceVersion: "19127" uid: d06a0a22-f1fb-4622-9466-a849a3042a7f kind: Event lastTimestamp: "2026-07-01T14:57:31Z" message: 'Failed to update status for InferenceService "isvc-primary-df069b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-df069b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:57:31Z" name: isvc-primary-df069b.18be3256f0ce50fe namespace: kserve-ci-e2e-test resourceVersion: "19158" uid: b7e583a1-1cb0-4168-976a-982dfabc54df reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:57:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-df069b namespace: kserve-ci-e2e-test resourceVersion: "19127" uid: d06a0a22-f1fb-4622-9466-a849a3042a7f kind: Event lastTimestamp: "2026-07-01T14:57:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-primary-df069b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:57:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:57:31Z" name: isvc-primary-df069b.18be3256f0ce89b6 namespace: kserve-ci-e2e-test resourceVersion: "19159" uid: b1181237-cd72-4e7e-83c3-d1ef4e5102fa reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:58:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-primary-df069b namespace: kserve-ci-e2e-test resourceVersion: "19768" uid: d06a0a22-f1fb-4622-9466-a849a3042a7f kind: Event lastTimestamp: "2026-07-01T14:58:53Z" message: InferenceService [isvc-primary-df069b] is Ready metadata: creationTimestamp: "2026-07-01T14:58:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:58:53Z" name: isvc-primary-df069b.18be3269fb86c141 namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 4152e7ac-8442-49aa-9fd3-caf896386e02 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:47:25.069528Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp to ip-10-0-134-20.ec2.internal metadata: creationTimestamp: "2026-07-01T14:47:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:47:25Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31c9c049b16e namespace: kserve-ci-e2e-test resourceVersion: "13954" uid: 0c20cb78-107e-4d3c-8f42-1d1ebb37cee8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-655cd54c75-lthr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:47:25Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13959" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:47:25Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:47:25Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31c9d9f1425a namespace: kserve-ci-e2e-test resourceVersion: "13972" uid: 67b7260a-b79b-4626-9dc4-0f7a83b66a2a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:47:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:47:25Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:32825628ffab44a0db6a86e1f74488e47dcab508ff646215bea8b81168e85657" already present on machine metadata: creationTimestamp: "2026-07-01T14:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:47:25Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31c9db98aa22 namespace: kserve-ci-e2e-test resourceVersion: "13974" uid: 31f2f87d-6e12-4ccd-98f5-f49159811d57 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:47:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:47:25Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T14:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:47:25Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31c9de0940c4 namespace: kserve-ci-e2e-test resourceVersion: "13975" uid: b5424665-d1ba-4c04-9f27-a0c897bd2451 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:47:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:47:25Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T14:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:47:25Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31c9de758512 namespace: kserve-ci-e2e-test resourceVersion: "13976" uid: b8514857-5aaa-4867-a14c-94488b75e39c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:47:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:47:29Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" metadata: creationTimestamp: "2026-07-01T14:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:47:29Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31cac57782fe namespace: kserve-ci-e2e-test resourceVersion: "14007" uid: 8858958d-4ef9-41f2-969f-fda43282b887 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:47:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:47:45Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" in 15.67s (15.67s including waiting). Image size: 1555870134 bytes.' metadata: creationTimestamp: "2026-07-01T14:47:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:47:45Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31ce6b8618f6 namespace: kserve-ci-e2e-test resourceVersion: "14122" uid: a316ece2-6a98-4d73-89c4-e635047ee2f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:47:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:47:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T14:47:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:47:45Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31ce6e9dc61a namespace: kserve-ci-e2e-test resourceVersion: "14123" uid: 82936c97-fe12-43dd-ab16-011ea6fdf6d7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:47:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:47:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T14:47:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:47:45Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31ce6f1074c2 namespace: kserve-ci-e2e-test resourceVersion: "14124" uid: 8eef0edb-ad6f-483e-ae55-20ea2632d123 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:47:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:47:45Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-01T14:47:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:47:45Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31ce6f1e01f2 namespace: kserve-ci-e2e-test resourceVersion: "14125" uid: 51f72672-300e-469a-9eb3-8390f1183f9e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:47:47Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.321s (2.321s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-01T14:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:47:47Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31cef97bda0b namespace: kserve-ci-e2e-test resourceVersion: "14140" uid: e73288f5-e850-4a9f-9dd3-f81828bfc6d6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:47:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T14:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:47:47Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31cefda2fb96 namespace: kserve-ci-e2e-test resourceVersion: "14141" uid: cc8aeb1c-629f-436c-9c2c-e148e84a8b45 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:47:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:47:47Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31cefe31d4c3 namespace: kserve-ci-e2e-test resourceVersion: "14142" uid: 1ed4c2d9-a192-4b3c-a22b-ddbb1448d211 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:47:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:47:47Z" message: Pulling image "quay.io/opendatahub/kserve-agent@sha256:b968ed73a63cc06f7467120ce75d205a47a53bf36fad1b5d010a25df8b9285c9" metadata: creationTimestamp: "2026-07-01T14:47:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:47:47Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31cefe402fcd namespace: kserve-ci-e2e-test resourceVersion: "14143" uid: ab0afb0b-6867-4220-b06f-88f76bc44da6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:47:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:47:51Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-agent@sha256:b968ed73a63cc06f7467120ce75d205a47a53bf36fad1b5d010a25df8b9285c9" in 3.58s (3.58s including waiting). Image size: 239994492 bytes.' metadata: creationTimestamp: "2026-07-01T14:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:47:51Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31cfd3a61695 namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: 7106b296-8a83-4cdf-9c0c-6e3afe6158fc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:47:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:47:51Z" message: 'Created container: agent' metadata: creationTimestamp: "2026-07-01T14:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:47:51Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31cfd992a562 namespace: kserve-ci-e2e-test resourceVersion: "14170" uid: 80d8cadb-d227-4f94-a6ed-275812ac8902 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:47:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:47:51Z" message: Started container agent metadata: creationTimestamp: "2026-07-01T14:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:47:51Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31cfdaa44934 namespace: kserve-ci-e2e-test resourceVersion: "14172" uid: 88eb025d-2f86-439d-a76b-bb25077321e5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T14:47:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:49:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:24Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31cfebbcc628 namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: 07f424e8-bada-42c5-8f3e-5639c1d25b63 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T14:47:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:49:14Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T14:47:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:14Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31d027896e9c namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: 79fca3c7-7d40-4b34-81a5-fe2ee32d8344 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:10Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31e23b2eaf8f namespace: kserve-ci-e2e-test resourceVersion: "14798" uid: 52de859f-ad49-4ea5-a3af-160795db8678 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{agent} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: Stopping container agent metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:10Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31e23b2ee7c2 namespace: kserve-ci-e2e-test resourceVersion: "14802" uid: d8c1da4b-49c1-464e-8fca-7c74e589c494 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:10Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31e23b2f2c3f namespace: kserve-ci-e2e-test resourceVersion: "14807" uid: e1ca805e-dd94-4df6-895b-98dc949769b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-01T14:49:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: d6b0e650-2b56-4cbe-bbc5-d23340095e08 kind: Event lastTimestamp: "2026-07-01T14:49:22Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:49:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:22Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp.18be31e2c7e3d09f namespace: kserve-ci-e2e-test resourceVersion: "14977" uid: 4d9acd48-ea7a-4cb7-be9d-7f6b1379e0a9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:47:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b namespace: kserve-ci-e2e-test resourceVersion: "13936" uid: f13ca73c-5422-4a16-802f-5d9baeab0681 kind: Event lastTimestamp: "2026-07-01T14:47:25Z" message: 'Created pod: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b-tgzlp' metadata: creationTimestamp: "2026-07-01T14:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:47:25Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b.18be31c9bfd1cda3 namespace: kserve-ci-e2e-test resourceVersion: "13951" uid: 115baeb5-c870-4d2a-91aa-fa71d13e4ed0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:47:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-raw-sklearn-batcher-2d7fd-predictor namespace: kserve-ci-e2e-test resourceVersion: "13935" uid: bb5af748-295d-4d25-81da-c73c4696fb55 kind: Event lastTimestamp: "2026-07-01T14:47:24Z" message: Scaled up replica set isvc-raw-sklearn-batcher-2d7fd-predictor-68d48f864b from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:47:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:47:24Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor.18be31c9b7116e42 namespace: kserve-ci-e2e-test resourceVersion: "13938" uid: 62a12659-2985-42c6-8e22-5dbb4de67e14 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:47:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-2d7fd-predictor namespace: kserve-ci-e2e-test resourceVersion: "13943" uid: fd43771d-0dad-484c-b811-f43ba5cece35 kind: Event lastTimestamp: "2026-07-01T14:47:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:47:55Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor.18be31cd3d03db98 namespace: kserve-ci-e2e-test resourceVersion: "14211" uid: 72006e69-2dff-466d-b049-eecc2a3401e3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:47:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-2d7fd-predictor namespace: kserve-ci-e2e-test resourceVersion: "13943" uid: fd43771d-0dad-484c-b811-f43ba5cece35 kind: Event lastTimestamp: "2026-07-01T14:47:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:47:55Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor.18be31cd3d895c44 namespace: kserve-ci-e2e-test resourceVersion: "14212" uid: c60b245f-7a28-4d93-a8a4-6143672af84c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T14:48:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-2d7fd-predictor namespace: kserve-ci-e2e-test resourceVersion: "14083" uid: fd43771d-0dad-484c-b811-f43ba5cece35 kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:48:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:10Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor.18be31d43c09bf9b namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 9cb346bd-d252-4299-8c25-9a50c2687e20 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T14:48:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-raw-sklearn-batcher-2d7fd-predictor namespace: kserve-ci-e2e-test resourceVersion: "14083" uid: fd43771d-0dad-484c-b811-f43ba5cece35 kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:48:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:10Z" name: isvc-raw-sklearn-batcher-2d7fd-predictor.18be31d43c84bfa7 namespace: kserve-ci-e2e-test resourceVersion: "14789" uid: 2047894e-cb59-4b50-824f-2ecb4e8233d5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T14:47:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-2d7fd namespace: kserve-ci-e2e-test resourceVersion: "13927" uid: a131f53e-f936-4746-be25-3440af6013be kind: Event lastTimestamp: "2026-07-01T14:49:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:49:04Z" name: isvc-raw-sklearn-batcher-2d7fd.18be31c9bd102792 namespace: kserve-ci-e2e-test resourceVersion: "14711" uid: f4ff6486-7616-4028-97fc-4347a9fe144e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:47:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-2d7fd namespace: kserve-ci-e2e-test resourceVersion: "13927" uid: a131f53e-f936-4746-be25-3440af6013be kind: Event lastTimestamp: "2026-07-01T14:47:25Z" message: 'Failed to update status for InferenceService "isvc-raw-sklearn-batcher-2d7fd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-2d7fd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:47:25Z" name: isvc-raw-sklearn-batcher-2d7fd.18be31c9bd96c1c9 namespace: kserve-ci-e2e-test resourceVersion: "13962" uid: a413b854-818f-47b0-8d2a-ce7944b5d10f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:47:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-2d7fd namespace: kserve-ci-e2e-test resourceVersion: "13927" uid: a131f53e-f936-4746-be25-3440af6013be kind: Event lastTimestamp: "2026-07-01T14:47:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-raw-sklearn-batcher-2d7fd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:47:25Z" name: isvc-raw-sklearn-batcher-2d7fd.18be31c9bd971f3d namespace: kserve-ci-e2e-test resourceVersion: "13963" uid: e0d82016-e0e8-4b99-895a-a789c439efa0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-raw-sklearn-batcher-2d7fd namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: a131f53e-f936-4746-be25-3440af6013be kind: Event lastTimestamp: "2026-07-01T14:49:04Z" message: InferenceService [isvc-raw-sklearn-batcher-2d7fd] is Ready metadata: creationTimestamp: "2026-07-01T14:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:49:04Z" name: isvc-raw-sklearn-batcher-2d7fd.18be31e0f1df13b6 namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: e4d2d0ed-57aa-48e8-85fc-4a1cc7620fa1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:59:01.767984Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-df069b-predictor-67966c757b-g9d6r namespace: kserve-ci-e2e-test resourceVersion: "19853" uid: 0b8d4451-d90b-4ba6-a935-916d55d1419d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-secondary-df069b-predictor-67966c757b-g9d6r to ip-10-0-134-20.ec2.internal metadata: creationTimestamp: "2026-07-01T14:59:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:59:01Z" name: isvc-secondary-df069b-predictor-67966c757b-g9d6r.18be326bf6c0725c namespace: kserve-ci-e2e-test resourceVersion: "19857" uid: 4d0144b5-0860-43a8-9783-d24b8ae6ef27 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-655cd54c75-lthr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-secondary-df069b-predictor-67966c757b-g9d6r namespace: kserve-ci-e2e-test resourceVersion: "19861" uid: 0b8d4451-d90b-4ba6-a935-916d55d1419d kind: Event lastTimestamp: "2026-07-01T14:59:02Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:59:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:59:02Z" name: isvc-secondary-df069b-predictor-67966c757b-g9d6r.18be326c1054e824 namespace: kserve-ci-e2e-test resourceVersion: "19871" uid: 309c7c41-863b-400c-a633-9274838f70af reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:59:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-df069b-predictor-67966c757b-g9d6r namespace: kserve-ci-e2e-test resourceVersion: "19855" uid: 0b8d4451-d90b-4ba6-a935-916d55d1419d kind: Event lastTimestamp: "2026-07-01T14:59:05Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:32825628ffab44a0db6a86e1f74488e47dcab508ff646215bea8b81168e85657" already present on machine metadata: creationTimestamp: "2026-07-01T14:59:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:05Z" name: isvc-secondary-df069b-predictor-67966c757b-g9d6r.18be326c123f6e8b namespace: kserve-ci-e2e-test resourceVersion: "19908" uid: d4851525-bf4f-43d6-b59b-1d02d182487c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:59:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-df069b-predictor-67966c757b-g9d6r namespace: kserve-ci-e2e-test resourceVersion: "19855" uid: 0b8d4451-d90b-4ba6-a935-916d55d1419d kind: Event lastTimestamp: "2026-07-01T14:59:05Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T14:59:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:05Z" name: isvc-secondary-df069b-predictor-67966c757b-g9d6r.18be326c15bae7c1 namespace: kserve-ci-e2e-test resourceVersion: "19910" uid: c8c2d157-73d6-4381-be5e-f97a65f9323e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:59:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-df069b-predictor-67966c757b-g9d6r namespace: kserve-ci-e2e-test resourceVersion: "19855" uid: 0b8d4451-d90b-4ba6-a935-916d55d1419d kind: Event lastTimestamp: "2026-07-01T14:59:05Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T14:59:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:05Z" name: isvc-secondary-df069b-predictor-67966c757b-g9d6r.18be326c162edfee namespace: kserve-ci-e2e-test resourceVersion: "19911" uid: 72b42c82-717a-4899-a4cf-28cfde78b830 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-secondary-df069b-predictor-67966c757b-g9d6r namespace: kserve-ci-e2e-test resourceVersion: "19855" uid: 0b8d4451-d90b-4ba6-a935-916d55d1419d kind: Event lastTimestamp: "2026-07-01T14:59:10Z" message: Back-off restarting failed container storage-initializer in pod isvc-secondary-df069b-predictor-67966c757b-g9d6r_kserve-ci-e2e-test(0b8d4451-d90b-4ba6-a935-916d55d1419d) metadata: creationTimestamp: "2026-07-01T14:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:10Z" name: isvc-secondary-df069b-predictor-67966c757b-g9d6r.18be326e0208adaa namespace: kserve-ci-e2e-test resourceVersion: "19948" uid: c3a1788a-68d9-4abf-ab48-674221f7f80a reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-secondary-df069b-predictor-67966c757b namespace: kserve-ci-e2e-test resourceVersion: "19843" uid: cf8eaba4-c290-443c-9fab-f3f2f4682f53 kind: Event lastTimestamp: "2026-07-01T14:59:01Z" message: 'Created pod: isvc-secondary-df069b-predictor-67966c757b-g9d6r' metadata: creationTimestamp: "2026-07-01T14:59:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:59:01Z" name: isvc-secondary-df069b-predictor-67966c757b.18be326bf6467b4c namespace: kserve-ci-e2e-test resourceVersion: "19854" uid: 618f50ea-b709-4fce-b37e-19371802ddfd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-secondary-df069b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19841" uid: f745b329-ec3c-4012-97b6-268da9902095 kind: Event lastTimestamp: "2026-07-01T14:59:01Z" message: Scaled up replica set isvc-secondary-df069b-predictor-67966c757b from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:59:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:59:01Z" name: isvc-secondary-df069b-predictor.18be326bf187a29e namespace: kserve-ci-e2e-test resourceVersion: "19844" uid: 06f75c86-5e97-481f-8c43-4bf4b10546d8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-df069b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19842" uid: d6471d28-99ab-4ca5-ae0c-d4e1add4a705 kind: Event lastTimestamp: "2026-07-01T14:59:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:59:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:59:16Z" name: isvc-secondary-df069b-predictor.18be326f703d0bff namespace: kserve-ci-e2e-test resourceVersion: "19998" uid: 02b484a5-0b2b-4643-8681-1cc521fdecb1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-secondary-df069b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19842" uid: d6471d28-99ab-4ca5-ae0c-d4e1add4a705 kind: Event lastTimestamp: "2026-07-01T14:59:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:59:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:59:16Z" name: isvc-secondary-df069b-predictor.18be326f70be0363 namespace: kserve-ci-e2e-test resourceVersion: "20000" uid: f869fd47-0bf1-4ae0-a7a6-de27ff3a5559 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 13 eventTime: null firstTimestamp: "2026-07-01T14:59:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-df069b namespace: kserve-ci-e2e-test resourceVersion: "19834" uid: 7de901fe-2f15-4d6f-bfb3-7e6ec4688407 kind: Event lastTimestamp: "2026-07-01T14:59:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:59:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:59:10Z" name: isvc-secondary-df069b.18be326bf1ce6244 namespace: kserve-ci-e2e-test resourceVersion: "19953" uid: d80d0706-fd95-40e4-a8bb-ef8742761aea reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:59:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-df069b namespace: kserve-ci-e2e-test resourceVersion: "19834" uid: 7de901fe-2f15-4d6f-bfb3-7e6ec4688407 kind: Event lastTimestamp: "2026-07-01T14:59:01Z" message: 'Failed to update status for InferenceService "isvc-secondary-df069b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-df069b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:59:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:59:01Z" name: isvc-secondary-df069b.18be326bf2254699 namespace: kserve-ci-e2e-test resourceVersion: "19863" uid: 6da8d204-9de6-43e1-9dbd-642658889d1d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:59:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-secondary-df069b namespace: kserve-ci-e2e-test resourceVersion: "19834" uid: 7de901fe-2f15-4d6f-bfb3-7e6ec4688407 kind: Event lastTimestamp: "2026-07-01T14:59:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-secondary-df069b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:59:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:59:01Z" name: isvc-secondary-df069b.18be326bf225a2b0 namespace: kserve-ci-e2e-test resourceVersion: "19864" uid: 31f989df-229e-47a5-ad75-13a853377a06 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:49:10.230998Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6 namespace: kserve-ci-e2e-test resourceVersion: "14799" uid: 22c2dede-364e-423f-938e-94b314e7f815 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6 to ip-10-0-134-20.ec2.internal metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:49:10Z" name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6.18be31e23c66580c namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: 9810b7d7-28e9-4acd-8f3e-9fbe971fe13c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-655cd54c75-lthr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6 namespace: kserve-ci-e2e-test resourceVersion: "14820" uid: 22c2dede-364e-423f-938e-94b314e7f815 kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:49:10Z" name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6.18be31e255e6d126 namespace: kserve-ci-e2e-test resourceVersion: "14851" uid: 2933b898-d1d6-4624-9262-a8a31ced2cf1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6 namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: 22c2dede-364e-423f-938e-94b314e7f815 kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:32825628ffab44a0db6a86e1f74488e47dcab508ff646215bea8b81168e85657" already present on machine metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:10Z" name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6.18be31e257b40f7b namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: b7a7921b-4b17-4502-94ae-6de6fc4f2ebd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6 namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: 22c2dede-364e-423f-938e-94b314e7f815 kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:10Z" name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6.18be31e25ab0672b namespace: kserve-ci-e2e-test resourceVersion: "14855" uid: 7f25c68d-143d-46d9-9885-32310aa6b8b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6 namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: 22c2dede-364e-423f-938e-94b314e7f815 kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:10Z" name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6.18be31e25b266fb1 namespace: kserve-ci-e2e-test resourceVersion: "14856" uid: 11f303ed-e3f1-4221-affb-9768fa81d12a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6 namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: 22c2dede-364e-423f-938e-94b314e7f815 kind: Event lastTimestamp: "2026-07-01T14:49:14Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T14:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:14Z" name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6.18be31e353fc1396 namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: e59e13b1-00f4-4058-b588-548b96d2a1be reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6 namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: 22c2dede-364e-423f-938e-94b314e7f815 kind: Event lastTimestamp: "2026-07-01T14:49:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T14:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:14Z" name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6.18be31e3578b01eb namespace: kserve-ci-e2e-test resourceVersion: "14914" uid: 9014630b-e6d5-4d2b-a670-19feda881885 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6 namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: 22c2dede-364e-423f-938e-94b314e7f815 kind: Event lastTimestamp: "2026-07-01T14:49:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T14:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:14Z" name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6.18be31e35800a470 namespace: kserve-ci-e2e-test resourceVersion: "14915" uid: 02a87490-5cbe-41e2-bab9-48a944c56433 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6 namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: 22c2dede-364e-423f-938e-94b314e7f815 kind: Event lastTimestamp: "2026-07-01T14:49:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T14:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:14Z" name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6.18be31e3580b00c1 namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: e57968c4-7781-4a74-aa57-b6aee6a40751 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6 namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: 22c2dede-364e-423f-938e-94b314e7f815 kind: Event lastTimestamp: "2026-07-01T14:49:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T14:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:15Z" name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6.18be31e35ac37dd7 namespace: kserve-ci-e2e-test resourceVersion: "14920" uid: de9b63b9-54ec-4045-9991-473b3c1a5839 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6 namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: 22c2dede-364e-423f-938e-94b314e7f815 kind: Event lastTimestamp: "2026-07-01T14:49:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:15Z" name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6.18be31e35b2fe531 namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 35fbd361-cf1f-4ac1-a81e-c6c240ef2674 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T14:49:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6 namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: 22c2dede-364e-423f-938e-94b314e7f815 kind: Event lastTimestamp: "2026-07-01T14:51:02Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:02Z" name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6.18be31e390f0b7bf namespace: kserve-ci-e2e-test resourceVersion: "15924" uid: 26799606-1506-4024-9748-c2bfdd9d8e97 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6 namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: 22c2dede-364e-423f-938e-94b314e7f815 kind: Event lastTimestamp: "2026-07-01T14:51:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T14:51:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:00Z" name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6.18be31fbf078f4f3 namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: 5d7a2853-cb09-4d65-9d64-ce20df1940d6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6 namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: 22c2dede-364e-423f-938e-94b314e7f815 kind: Event lastTimestamp: "2026-07-01T14:51:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:51:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:00Z" name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6.18be31fbf07a0da0 namespace: kserve-ci-e2e-test resourceVersion: "15829" uid: dcd77a1f-32eb-470f-9ce2-7879b387c202 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6 namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: 22c2dede-364e-423f-938e-94b314e7f815 kind: Event lastTimestamp: "2026-07-01T14:51:02Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:51:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:02Z" name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6.18be31fc7b364984 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: f089adf2-ccc7-4645-a2ac-c343b1ca611a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b namespace: kserve-ci-e2e-test resourceVersion: "14769" uid: 292277da-bd5e-436e-88bf-b340f1aa3de5 kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: 'Created pod: isvc-sklearn-graph-raw-63a63-predictor-8b657594b-jsqm6' metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:10Z" name: isvc-sklearn-graph-raw-63a63-predictor-8b657594b.18be31e23bd2ace3 namespace: kserve-ci-e2e-test resourceVersion: "14804" uid: c48fdb9f-8016-4f2a-9ae3-3196b6a21fd0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-63a63-predictor namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 0e74f402-a237-42b7-91cf-d454dc2f69f3 kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: Scaled up replica set isvc-sklearn-graph-raw-63a63-predictor-8b657594b from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:10Z" name: isvc-sklearn-graph-raw-63a63-predictor.18be31e235fbb4de namespace: kserve-ci-e2e-test resourceVersion: "14770" uid: 2bb41a13-1c60-414c-aa0c-293582ef5674 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:49:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-63a63-predictor namespace: kserve-ci-e2e-test resourceVersion: "14768" uid: d30688c8-bef3-4892-b506-1e61d2b765c6 kind: Event lastTimestamp: "2026-07-01T14:49:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:40Z" name: isvc-sklearn-graph-raw-63a63-predictor.18be31e5b4ceca14 namespace: kserve-ci-e2e-test resourceVersion: "15119" uid: f663c8f1-a162-450d-bfa2-278c120987b6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:49:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-63a63-predictor namespace: kserve-ci-e2e-test resourceVersion: "14768" uid: d30688c8-bef3-4892-b506-1e61d2b765c6 kind: Event lastTimestamp: "2026-07-01T14:49:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:40Z" name: isvc-sklearn-graph-raw-63a63-predictor.18be31e5b55e3b6c namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: 98b5bdf4-7501-451b-bc02-9abecaa698f3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T14:49:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-63a63-predictor namespace: kserve-ci-e2e-test resourceVersion: "15002" uid: d30688c8-bef3-4892-b506-1e61d2b765c6 kind: Event lastTimestamp: "2026-07-01T14:50:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:55Z" name: isvc-sklearn-graph-raw-63a63-predictor.18be31ecb3daa2d3 namespace: kserve-ci-e2e-test resourceVersion: "15716" uid: 81c330a4-4b94-4aa3-983d-86f0cec59dd1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T14:49:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-63a63-predictor namespace: kserve-ci-e2e-test resourceVersion: "15002" uid: d30688c8-bef3-4892-b506-1e61d2b765c6 kind: Event lastTimestamp: "2026-07-01T14:50:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:55Z" name: isvc-sklearn-graph-raw-63a63-predictor.18be31ecb45ff5c9 namespace: kserve-ci-e2e-test resourceVersion: "15717" uid: d9a21cf1-dab5-46c7-9ffa-15720b2ea442 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-63a63 namespace: kserve-ci-e2e-test resourceVersion: "14757" uid: 3f583135-77ba-4c0b-b3b3-c31a8cc9e50e kind: Event lastTimestamp: "2026-07-01T14:50:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:33Z" name: isvc-sklearn-graph-raw-63a63.18be31e2360a42ce namespace: kserve-ci-e2e-test resourceVersion: "15518" uid: d7ccf936-bd4f-4f6d-9d50-351353ae0558 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-63a63 namespace: kserve-ci-e2e-test resourceVersion: "14757" uid: 3f583135-77ba-4c0b-b3b3-c31a8cc9e50e kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-63a63": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-63a63": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:49:10Z" name: isvc-sklearn-graph-raw-63a63.18be31e2368ffc21 namespace: kserve-ci-e2e-test resourceVersion: "14773" uid: 29231931-8f1d-4af8-b261-f9221d7cb6e0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-63a63 namespace: kserve-ci-e2e-test resourceVersion: "15516" uid: 3f583135-77ba-4c0b-b3b3-c31a8cc9e50e kind: Event lastTimestamp: "2026-07-01T14:50:33Z" message: InferenceService [isvc-sklearn-graph-raw-63a63] is Ready metadata: creationTimestamp: "2026-07-01T14:50:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:33Z" name: isvc-sklearn-graph-raw-63a63.18be31f58715d937 namespace: kserve-ci-e2e-test resourceVersion: "15517" uid: 703249e3-d053-4dbf-963f-4577f3d858ba reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:51:00.604600Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4 namespace: kserve-ci-e2e-test resourceVersion: "15816" uid: b056f5a5-627b-4263-8487-0bff33f86e6e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4 to ip-10-0-134-20.ec2.internal metadata: creationTimestamp: "2026-07-01T14:51:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:51:00Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4.18be31fbef2dd753 namespace: kserve-ci-e2e-test resourceVersion: "15821" uid: 8914a61e-d59a-4d4e-9665-163b9eb9ac14 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-655cd54c75-lthr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4 namespace: kserve-ci-e2e-test resourceVersion: "15831" uid: b056f5a5-627b-4263-8487-0bff33f86e6e kind: Event lastTimestamp: "2026-07-01T14:51:01Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:51:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:51:01Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4.18be31fc0862227c namespace: kserve-ci-e2e-test resourceVersion: "15888" uid: f3ed54af-045e-420a-a598-8f54f60191b8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: b056f5a5-627b-4263-8487-0bff33f86e6e kind: Event lastTimestamp: "2026-07-01T14:51:01Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:32825628ffab44a0db6a86e1f74488e47dcab508ff646215bea8b81168e85657" already present on machine metadata: creationTimestamp: "2026-07-01T14:51:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:01Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4.18be31fc0a0bb1a7 namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: a965d119-b100-436c-a68d-480e839098a1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: b056f5a5-627b-4263-8487-0bff33f86e6e kind: Event lastTimestamp: "2026-07-01T14:51:01Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T14:51:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:01Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4.18be31fc0ddf0c35 namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: e4adf725-6316-4672-abed-42e4f1190a75 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: b056f5a5-627b-4263-8487-0bff33f86e6e kind: Event lastTimestamp: "2026-07-01T14:51:01Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T14:51:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:01Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4.18be31fc0e4814d4 namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: 818276e7-f0cb-4e8c-b124-4fc83586a3b7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: b056f5a5-627b-4263-8487-0bff33f86e6e kind: Event lastTimestamp: "2026-07-01T14:51:05Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T14:51:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:05Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4.18be31fd0f2f7da7 namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: e960b65a-843d-4429-859c-8c7ef9f0441b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: b056f5a5-627b-4263-8487-0bff33f86e6e kind: Event lastTimestamp: "2026-07-01T14:51:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T14:51:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:05Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4.18be31fd129a7be0 namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: c0440440-a4d5-43c7-bf21-155340b39273 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: b056f5a5-627b-4263-8487-0bff33f86e6e kind: Event lastTimestamp: "2026-07-01T14:51:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T14:51:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:05Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4.18be31fd1373c337 namespace: kserve-ci-e2e-test resourceVersion: "15957" uid: 1a7a09c3-c70e-4b4a-9f88-bd513a5a8c3d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: b056f5a5-627b-4263-8487-0bff33f86e6e kind: Event lastTimestamp: "2026-07-01T14:51:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T14:51:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:05Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4.18be31fd137efa08 namespace: kserve-ci-e2e-test resourceVersion: "15958" uid: 8b1068e1-5fcd-485a-a399-26c2a3c8fdeb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: b056f5a5-627b-4263-8487-0bff33f86e6e kind: Event lastTimestamp: "2026-07-01T14:51:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T14:51:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:05Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4.18be31fd163d8d0c namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 23466e13-d801-4752-88ab-5db5bfcae886 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: b056f5a5-627b-4263-8487-0bff33f86e6e kind: Event lastTimestamp: "2026-07-01T14:51:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:51:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:05Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4.18be31fd16a86313 namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: a1e79c03-ba29-4797-b05a-dcae46ab514a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T14:51:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: b056f5a5-627b-4263-8487-0bff33f86e6e kind: Event lastTimestamp: "2026-07-01T14:52:52Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:51:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:52Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4.18be31fd4b9e5886 namespace: kserve-ci-e2e-test resourceVersion: "16909" uid: 1721050b-d63b-4e5b-a08f-29d61288c4b5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: b056f5a5-627b-4263-8487-0bff33f86e6e kind: Event lastTimestamp: "2026-07-01T14:52:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T14:52:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:51Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4.18be3215aadf6b11 namespace: kserve-ci-e2e-test resourceVersion: "16869" uid: 215130ad-6081-48ce-9f92-47900ce5c8f1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: b056f5a5-627b-4263-8487-0bff33f86e6e kind: Event lastTimestamp: "2026-07-01T14:52:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:52:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:51Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4.18be3215aadfd302 namespace: kserve-ci-e2e-test resourceVersion: "16870" uid: 822303ea-0f11-450b-b911-8b3669de6b0a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: b056f5a5-627b-4263-8487-0bff33f86e6e kind: Event lastTimestamp: "2026-07-01T14:52:52Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:52Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4.18be3215fa45feb9 namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: f538466f-1c65-4d87-ba9f-5e667b4583f8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6 namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: ae92da85-536b-44e6-b861-65f4f20bb89d kind: Event lastTimestamp: "2026-07-01T14:51:00Z" message: 'Created pod: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6-x5bg4' metadata: creationTimestamp: "2026-07-01T14:51:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:00Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6.18be31fbee889f6d namespace: kserve-ci-e2e-test resourceVersion: "15818" uid: 13318e46-57ec-4c46-a5dd-442523baf290 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-raw-hpa-02737-predictor namespace: kserve-ci-e2e-test resourceVersion: "15788" uid: b58a681a-8ce9-44fe-b04c-424476e4da52 kind: Event lastTimestamp: "2026-07-01T14:51:00Z" message: Scaled up replica set isvc-sklearn-graph-raw-hpa-02737-predictor-b7bc4b6b6 from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:51:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:00Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor.18be31fbe88bd22b namespace: kserve-ci-e2e-test resourceVersion: "15792" uid: 3d5b2b4c-e465-473e-bb1b-df558c9adf1c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-02737-predictor namespace: kserve-ci-e2e-test resourceVersion: "15791" uid: 2163348e-348b-4556-bd3b-16ab60f198d8 kind: Event lastTimestamp: "2026-07-01T14:51:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:51:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:15Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor.18be31ff67bf0526 namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 0263d770-87a4-4ad6-a84a-e886a4562b3a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-02737-predictor namespace: kserve-ci-e2e-test resourceVersion: "15791" uid: 2163348e-348b-4556-bd3b-16ab60f198d8 kind: Event lastTimestamp: "2026-07-01T14:51:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:51:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:15Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor.18be31ff6895794c namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: 5c02d5f7-b044-493e-a345-5a03c39b822b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T14:51:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-02737-predictor namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 2163348e-348b-4556-bd3b-16ab60f198d8 kind: Event lastTimestamp: "2026-07-01T14:52:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:51:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:45Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor.18be3202e7f72feb namespace: kserve-ci-e2e-test resourceVersion: "16763" uid: 6a4a2138-c786-4cc4-946f-451ee5e39094 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T14:51:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-raw-hpa-02737-predictor namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 2163348e-348b-4556-bd3b-16ab60f198d8 kind: Event lastTimestamp: "2026-07-01T14:52:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:51:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:45Z" name: isvc-sklearn-graph-raw-hpa-02737-predictor.18be3202e8ac4f63 namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 3dcd7b17-3636-486e-96d6-f66dd6a2eea6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T14:51:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-02737 namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: af518630-f1e2-43f5-baf7-2bbde36fae22 kind: Event lastTimestamp: "2026-07-01T14:52:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:51:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:52:22Z" name: isvc-sklearn-graph-raw-hpa-02737.18be31fbe90807f0 namespace: kserve-ci-e2e-test resourceVersion: "16562" uid: b9461979-282c-4ed4-82fe-25157fd41148 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-02737 namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: af518630-f1e2-43f5-baf7-2bbde36fae22 kind: Event lastTimestamp: "2026-07-01T14:51:00Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-raw-hpa-02737": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-raw-hpa-02737": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:51:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:51:00Z" name: isvc-sklearn-graph-raw-hpa-02737.18be31fbe99a1453 namespace: kserve-ci-e2e-test resourceVersion: "15796" uid: a2614aae-ede8-4426-9a38-acc798ed2808 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-raw-hpa-02737 namespace: kserve-ci-e2e-test resourceVersion: "16560" uid: af518630-f1e2-43f5-baf7-2bbde36fae22 kind: Event lastTimestamp: "2026-07-01T14:52:22Z" message: InferenceService [isvc-sklearn-graph-raw-hpa-02737] is Ready metadata: creationTimestamp: "2026-07-01T14:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:52:22Z" name: isvc-sklearn-graph-raw-hpa-02737.18be320f0507bafd namespace: kserve-ci-e2e-test resourceVersion: "16561" uid: 30601b99-7b99-4c7a-a479-87cabb94eb63 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:54:46.312794Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m namespace: kserve-ci-e2e-test resourceVersion: "17863" uid: 807a9b97-34bf-4546-8f35-5d378f8806d9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m to ip-10-0-134-20.ec2.internal metadata: creationTimestamp: "2026-07-01T14:54:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:54:46Z" name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m.18be32307c6f96a1 namespace: kserve-ci-e2e-test resourceVersion: "17872" uid: d0d3eb69-48cc-45df-bcf1-fa11e87d2a4a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-655cd54c75-lthr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:54:46Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m namespace: kserve-ci-e2e-test resourceVersion: "17867" uid: 807a9b97-34bf-4546-8f35-5d378f8806d9 kind: Event lastTimestamp: "2026-07-01T14:54:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-scale-raw-b7013-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T14:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:54:46Z" name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m.18be3230872aa950 namespace: kserve-ci-e2e-test resourceVersion: "17900" uid: 52f88499-660c-413b-961e-93cea1f7aa72 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:54:47Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m namespace: kserve-ci-e2e-test resourceVersion: "17891" uid: 807a9b97-34bf-4546-8f35-5d378f8806d9 kind: Event lastTimestamp: "2026-07-01T14:54:47Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:54:47Z" name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m.18be3230b9be4ea4 namespace: kserve-ci-e2e-test resourceVersion: "17908" uid: 9c868585-108d-4285-a707-5373fff4b267 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:54:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m namespace: kserve-ci-e2e-test resourceVersion: "17867" uid: 807a9b97-34bf-4546-8f35-5d378f8806d9 kind: Event lastTimestamp: "2026-07-01T14:54:47Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:32825628ffab44a0db6a86e1f74488e47dcab508ff646215bea8b81168e85657" already present on machine metadata: creationTimestamp: "2026-07-01T14:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:54:47Z" name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m.18be3230bb44fb87 namespace: kserve-ci-e2e-test resourceVersion: "17910" uid: 055705e1-1089-45de-abd5-316747ac0b1b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:54:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m namespace: kserve-ci-e2e-test resourceVersion: "17867" uid: 807a9b97-34bf-4546-8f35-5d378f8806d9 kind: Event lastTimestamp: "2026-07-01T14:54:47Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T14:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:54:47Z" name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m.18be3230bf058359 namespace: kserve-ci-e2e-test resourceVersion: "17913" uid: 3f2f57db-5fd7-4586-a432-7b83959cde01 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:54:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m namespace: kserve-ci-e2e-test resourceVersion: "17867" uid: 807a9b97-34bf-4546-8f35-5d378f8806d9 kind: Event lastTimestamp: "2026-07-01T14:54:47Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T14:54:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:54:47Z" name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m.18be3230bf8c9f29 namespace: kserve-ci-e2e-test resourceVersion: "17915" uid: e84c73d3-b237-4a33-93f8-d316fc3b6f57 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:54:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m namespace: kserve-ci-e2e-test resourceVersion: "17867" uid: 807a9b97-34bf-4546-8f35-5d378f8806d9 kind: Event lastTimestamp: "2026-07-01T14:54:51Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T14:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:54:51Z" name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m.18be3231ae3c8b4d namespace: kserve-ci-e2e-test resourceVersion: "17949" uid: c75b1d58-fe36-450c-b585-ed74dc71d15e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:54:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m namespace: kserve-ci-e2e-test resourceVersion: "17867" uid: 807a9b97-34bf-4546-8f35-5d378f8806d9 kind: Event lastTimestamp: "2026-07-01T14:54:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T14:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:54:51Z" name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m.18be3231b183d307 namespace: kserve-ci-e2e-test resourceVersion: "17951" uid: aea25099-4c5f-4503-a111-9ae17fac2760 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:54:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m namespace: kserve-ci-e2e-test resourceVersion: "17867" uid: 807a9b97-34bf-4546-8f35-5d378f8806d9 kind: Event lastTimestamp: "2026-07-01T14:54:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T14:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:54:51Z" name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m.18be3231b1f73030 namespace: kserve-ci-e2e-test resourceVersion: "17952" uid: a7c84706-20a1-4506-aa6d-82f1c09dcf72 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:54:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m namespace: kserve-ci-e2e-test resourceVersion: "17867" uid: 807a9b97-34bf-4546-8f35-5d378f8806d9 kind: Event lastTimestamp: "2026-07-01T14:54:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T14:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:54:51Z" name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m.18be3231b200e5fc namespace: kserve-ci-e2e-test resourceVersion: "17953" uid: 81eb9ab6-878f-4df6-87cd-b8d6134d09bf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:54:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m namespace: kserve-ci-e2e-test resourceVersion: "17867" uid: 807a9b97-34bf-4546-8f35-5d378f8806d9 kind: Event lastTimestamp: "2026-07-01T14:54:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T14:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:54:51Z" name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m.18be3231b4d0223b namespace: kserve-ci-e2e-test resourceVersion: "17956" uid: 6742894b-22ac-4ca3-8610-a191635fb167 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:54:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m namespace: kserve-ci-e2e-test resourceVersion: "17867" uid: 807a9b97-34bf-4546-8f35-5d378f8806d9 kind: Event lastTimestamp: "2026-07-01T14:54:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:54:51Z" name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m.18be3231b542eae3 namespace: kserve-ci-e2e-test resourceVersion: "17957" uid: 7671e80d-b62c-4393-99b4-ce69c6259146 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 18 eventTime: null firstTimestamp: "2026-07-01T14:54:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m namespace: kserve-ci-e2e-test resourceVersion: "17867" uid: 807a9b97-34bf-4546-8f35-5d378f8806d9 kind: Event lastTimestamp: "2026-07-01T14:57:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:57:32Z" name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m.18be32322619faf8 namespace: kserve-ci-e2e-test resourceVersion: "19179" uid: 1ee7f283-6e9f-4640-ade4-ecbe09863571 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:57:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m namespace: kserve-ci-e2e-test resourceVersion: "17867" uid: 807a9b97-34bf-4546-8f35-5d378f8806d9 kind: Event lastTimestamp: "2026-07-01T14:57:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T14:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:57:26Z" name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m.18be3255c7291935 namespace: kserve-ci-e2e-test resourceVersion: "19087" uid: 896cdbf1-3ccf-415c-9103-0820f8c08d2c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:57:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m namespace: kserve-ci-e2e-test resourceVersion: "17867" uid: 807a9b97-34bf-4546-8f35-5d378f8806d9 kind: Event lastTimestamp: "2026-07-01T14:57:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:57:26Z" name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m.18be3255c729c144 namespace: kserve-ci-e2e-test resourceVersion: "19088" uid: b0aeb11b-5f50-4511-b6be-8e5c11d2a34e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:57:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m namespace: kserve-ci-e2e-test resourceVersion: "17867" uid: 807a9b97-34bf-4546-8f35-5d378f8806d9 kind: Event lastTimestamp: "2026-07-01T14:57:34Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:57:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:57:34Z" name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m.18be325678a45fe3 namespace: kserve-ci-e2e-test resourceVersion: "19194" uid: d72efe8e-4425-44c7-a760-c522cbca5d41 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:54:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8 namespace: kserve-ci-e2e-test resourceVersion: "17842" uid: 671ff18a-0893-4e42-8071-be82e427d893 kind: Event lastTimestamp: "2026-07-01T14:54:46Z" message: 'Created pod: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8-w2g4m' metadata: creationTimestamp: "2026-07-01T14:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:54:46Z" name: isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8.18be32307bed7ff9 namespace: kserve-ci-e2e-test resourceVersion: "17865" uid: 91d3c1a0-2670-480a-86d8-a68fdf6dd32c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:54:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-scale-raw-b7013-predictor namespace: kserve-ci-e2e-test resourceVersion: "17840" uid: 3a0d1d7d-5fcf-4b47-8bb2-bffbe555de52 kind: Event lastTimestamp: "2026-07-01T14:54:46Z" message: Scaled up replica set isvc-sklearn-scale-raw-b7013-predictor-5cc5b679b8 from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:54:46Z" name: isvc-sklearn-scale-raw-b7013-predictor.18be32307684b887 namespace: kserve-ci-e2e-test resourceVersion: "17843" uid: dffd04d3-d5bb-476d-a96e-5819e9c669b0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:55:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-b7013-predictor namespace: kserve-ci-e2e-test resourceVersion: "17841" uid: e18d1a4f-4046-430c-836c-52179ab0bc9a kind: Event lastTimestamp: "2026-07-01T14:55:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:55:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:55:01Z" name: isvc-sklearn-scale-raw-b7013-predictor.18be3233f595e845 namespace: kserve-ci-e2e-test resourceVersion: "18039" uid: 3b054cf8-a37f-41a5-a8cd-e3c8e5bbbcf2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:55:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-b7013-predictor namespace: kserve-ci-e2e-test resourceVersion: "17841" uid: e18d1a4f-4046-430c-836c-52179ab0bc9a kind: Event lastTimestamp: "2026-07-01T14:55:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:55:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:55:01Z" name: isvc-sklearn-scale-raw-b7013-predictor.18be3233f61ad6d4 namespace: kserve-ci-e2e-test resourceVersion: "18041" uid: 946d5c0a-a40e-4e13-aa14-ad0e422cb00b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T14:55:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-b7013-predictor namespace: kserve-ci-e2e-test resourceVersion: "18040" uid: e18d1a4f-4046-430c-836c-52179ab0bc9a kind: Event lastTimestamp: "2026-07-01T14:57:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:57:16Z" name: isvc-sklearn-scale-raw-b7013-predictor.18be32377504313d namespace: kserve-ci-e2e-test resourceVersion: "18988" uid: 53604f15-c0fd-44e5-8b95-f195e148793d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T14:55:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-scale-raw-b7013-predictor namespace: kserve-ci-e2e-test resourceVersion: "18040" uid: e18d1a4f-4046-430c-836c-52179ab0bc9a kind: Event lastTimestamp: "2026-07-01T14:57:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:57:16Z" name: isvc-sklearn-scale-raw-b7013-predictor.18be323775822e61 namespace: kserve-ci-e2e-test resourceVersion: "18990" uid: fac362a2-4210-4a6c-9943-5538423ba371 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T14:54:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-b7013 namespace: kserve-ci-e2e-test resourceVersion: "17829" uid: 4b3cf3bf-ae57-42cb-a7bd-832efff9cb5e kind: Event lastTimestamp: "2026-07-01T14:57:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:57:22Z" name: isvc-sklearn-scale-raw-b7013.18be32307686b78c namespace: kserve-ci-e2e-test resourceVersion: "19042" uid: af9a6be2-2489-462c-a08d-7d6f89eecfbf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:54:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-b7013 namespace: kserve-ci-e2e-test resourceVersion: "17829" uid: 4b3cf3bf-ae57-42cb-a7bd-832efff9cb5e kind: Event lastTimestamp: "2026-07-01T14:54:46Z" message: 'Failed to update status for InferenceService "isvc-sklearn-scale-raw-b7013": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-scale-raw-b7013": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:54:46Z" name: isvc-sklearn-scale-raw-b7013.18be3230771d061e namespace: kserve-ci-e2e-test resourceVersion: "17846" uid: 87c97c3e-de11-4224-a911-f6c03250b8e1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:57:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-scale-raw-b7013 namespace: kserve-ci-e2e-test resourceVersion: "19039" uid: 4b3cf3bf-ae57-42cb-a7bd-832efff9cb5e kind: Event lastTimestamp: "2026-07-01T14:57:22Z" message: InferenceService [isvc-sklearn-scale-raw-b7013] is Ready metadata: creationTimestamp: "2026-07-01T14:57:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:57:22Z" name: isvc-sklearn-scale-raw-b7013.18be3254d8678cf8 namespace: kserve-ci-e2e-test resourceVersion: "19040" uid: b4532b9b-08ac-41ca-80ec-f93c6715f1cf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:49:10.315292Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9 namespace: kserve-ci-e2e-test resourceVersion: "14824" uid: 45794eea-ae6e-4638-8ea4-db648638cf41 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9 to ip-10-0-134-20.ec2.internal metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:49:10Z" name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9.18be31e2416c89f7 namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: e28c901a-167e-4b20-89ef-f37428dff99d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-655cd54c75-lthr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: 45794eea-ae6e-4638-8ea4-db648638cf41 kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:49:10Z" name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9.18be31e25b58829b namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 12702e64-4186-437d-9f8c-3e9ce9b17570 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 45794eea-ae6e-4638-8ea4-db648638cf41 kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:32825628ffab44a0db6a86e1f74488e47dcab508ff646215bea8b81168e85657" already present on machine metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:10Z" name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9.18be31e25d28fc24 namespace: kserve-ci-e2e-test resourceVersion: "14859" uid: 1057622c-a580-48b4-a16d-e3a3fe0908c2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 45794eea-ae6e-4638-8ea4-db648638cf41 kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:10Z" name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9.18be31e25fd64b76 namespace: kserve-ci-e2e-test resourceVersion: "14860" uid: 5e5af878-cfe4-4b68-aece-d4de119aaa96 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 45794eea-ae6e-4638-8ea4-db648638cf41 kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:10Z" name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9.18be31e26046f0bb namespace: kserve-ci-e2e-test resourceVersion: "14861" uid: f23ed097-4802-4d60-8da1-456be84d73aa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 45794eea-ae6e-4638-8ea4-db648638cf41 kind: Event lastTimestamp: "2026-07-01T14:49:14Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-01T14:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:14Z" name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9.18be31e354390c00 namespace: kserve-ci-e2e-test resourceVersion: "14910" uid: 2f8555b0-6a7b-423f-903c-c6b2ded7ac2f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 45794eea-ae6e-4638-8ea4-db648638cf41 kind: Event lastTimestamp: "2026-07-01T14:49:32Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 17.658s (17.658s including waiting). Image size: 1305293052 bytes.' metadata: creationTimestamp: "2026-07-01T14:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:32Z" name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9.18be31e770c1d83a namespace: kserve-ci-e2e-test resourceVersion: "15052" uid: fe922f49-c132-484c-9934-2092e80dbc32 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 45794eea-ae6e-4638-8ea4-db648638cf41 kind: Event lastTimestamp: "2026-07-01T14:49:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T14:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:32Z" name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9.18be31e773c87ef1 namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: 375b9dd2-2a2b-4cc0-9c03-26bc0fd928ee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 45794eea-ae6e-4638-8ea4-db648638cf41 kind: Event lastTimestamp: "2026-07-01T14:49:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T14:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:32Z" name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9.18be31e7743991bf namespace: kserve-ci-e2e-test resourceVersion: "15055" uid: c66c4108-4f31-4c4a-8681-ea90f52e97ac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 45794eea-ae6e-4638-8ea4-db648638cf41 kind: Event lastTimestamp: "2026-07-01T14:49:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T14:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:32Z" name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9.18be31e774430cb8 namespace: kserve-ci-e2e-test resourceVersion: "15056" uid: 3e34a395-de4c-4b4f-9140-ad7be4f21547 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 45794eea-ae6e-4638-8ea4-db648638cf41 kind: Event lastTimestamp: "2026-07-01T14:49:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T14:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:32Z" name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9.18be31e7784b8b28 namespace: kserve-ci-e2e-test resourceVersion: "15058" uid: 0c58a9b3-196d-4aa7-8937-1444f52755cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 45794eea-ae6e-4638-8ea4-db648638cf41 kind: Event lastTimestamp: "2026-07-01T14:49:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:49:32Z" name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9.18be31e778c74177 namespace: kserve-ci-e2e-test resourceVersion: "15059" uid: 1c48319c-f326-46b0-af19-8328c3d5b855 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-01T14:49:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 45794eea-ae6e-4638-8ea4-db648638cf41 kind: Event lastTimestamp: "2026-07-01T14:50:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:49:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:30Z" name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9.18be31e7c70abb2a namespace: kserve-ci-e2e-test resourceVersion: "15488" uid: 0a0cd498-6263-40e6-b173-cc5a6b53bbd6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 45794eea-ae6e-4638-8ea4-db648638cf41 kind: Event lastTimestamp: "2026-07-01T14:51:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T14:51:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:00Z" name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9.18be31fbf856b3c9 namespace: kserve-ci-e2e-test resourceVersion: "15861" uid: 9744c683-311a-44eb-bf41-df0b05786466 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 45794eea-ae6e-4638-8ea4-db648638cf41 kind: Event lastTimestamp: "2026-07-01T14:51:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:51:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:00Z" name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9.18be31fbf857327a namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 6e92c49a-cf27-4e3a-aab9-86c8e77de072 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc namespace: kserve-ci-e2e-test resourceVersion: "14810" uid: ad949183-af77-4e35-99c7-597bdae31585 kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: 'Created pod: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc-xtcv9' metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:10Z" name: isvc-xgboost-graph-raw-63a63-predictor-5866499bcc.18be31e240cb1cbd namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 3ba3494e-fa1e-4861-bc2f-5f2e7f6ec589 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-63a63-predictor namespace: kserve-ci-e2e-test resourceVersion: "14803" uid: e8e51e06-fc4b-4159-bf1b-ce7768f81966 kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: Scaled up replica set isvc-xgboost-graph-raw-63a63-predictor-5866499bcc from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:10Z" name: isvc-xgboost-graph-raw-63a63-predictor.18be31e23cd9f059 namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: e76ab638-e0ca-4da7-9ed2-deaaf0e72a98 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:49:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-63a63-predictor namespace: kserve-ci-e2e-test resourceVersion: "14808" uid: 56dbaa55-4681-4f68-95fc-a8ca971c16eb kind: Event lastTimestamp: "2026-07-01T14:49:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:40Z" name: isvc-xgboost-graph-raw-63a63-predictor.18be31e5bb96bee4 namespace: kserve-ci-e2e-test resourceVersion: "15122" uid: 5bf3c123-8dbf-475d-88b4-7af2f47d1120 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:49:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-63a63-predictor namespace: kserve-ci-e2e-test resourceVersion: "14808" uid: 56dbaa55-4681-4f68-95fc-a8ca971c16eb kind: Event lastTimestamp: "2026-07-01T14:49:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:49:40Z" name: isvc-xgboost-graph-raw-63a63-predictor.18be31e5bc14086c namespace: kserve-ci-e2e-test resourceVersion: "15123" uid: c7cf962f-7dd1-4e19-82d2-5d9aaa454b31 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T14:49:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-63a63-predictor namespace: kserve-ci-e2e-test resourceVersion: "15005" uid: 56dbaa55-4681-4f68-95fc-a8ca971c16eb kind: Event lastTimestamp: "2026-07-01T14:50:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:55Z" name: isvc-xgboost-graph-raw-63a63-predictor.18be31ecb9ca8cd7 namespace: kserve-ci-e2e-test resourceVersion: "15720" uid: cc2cd88d-cee8-4382-a344-da062d03bc4f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-01T14:49:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-63a63-predictor namespace: kserve-ci-e2e-test resourceVersion: "15005" uid: 56dbaa55-4681-4f68-95fc-a8ca971c16eb kind: Event lastTimestamp: "2026-07-01T14:50:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:55Z" name: isvc-xgboost-graph-raw-63a63-predictor.18be31ecba3ffd42 namespace: kserve-ci-e2e-test resourceVersion: "15721" uid: 5c8203d0-38ca-4924-9455-50a7ba5cbb9d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-63a63 namespace: kserve-ci-e2e-test resourceVersion: "14775" uid: 976342e2-b9e0-4a94-85d1-446ea2c87d66 kind: Event lastTimestamp: "2026-07-01T14:50:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:40Z" name: isvc-xgboost-graph-raw-63a63.18be31e23ceb757e namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: 75f222f8-8d4f-4ec7-83ec-e4c627f18b6d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:49:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-63a63 namespace: kserve-ci-e2e-test resourceVersion: "14775" uid: 976342e2-b9e0-4a94-85d1-446ea2c87d66 kind: Event lastTimestamp: "2026-07-01T14:49:10Z" message: 'Failed to update status for InferenceService "isvc-xgboost-graph-raw-63a63": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-xgboost-graph-raw-63a63": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:49:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:49:10Z" name: isvc-xgboost-graph-raw-63a63.18be31e23d7dcbbc namespace: kserve-ci-e2e-test resourceVersion: "14817" uid: 660dc1ff-6d41-4d2e-a358-6487e9384d31 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-63a63 namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: 976342e2-b9e0-4a94-85d1-446ea2c87d66 kind: Event lastTimestamp: "2026-07-01T14:50:40Z" message: InferenceService [isvc-xgboost-graph-raw-63a63] is Ready metadata: creationTimestamp: "2026-07-01T14:50:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:40Z" name: isvc-xgboost-graph-raw-63a63.18be31f72b634cd6 namespace: kserve-ci-e2e-test resourceVersion: "15577" uid: 07a1e23c-7b69-4f69-9d76-254e393d620f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:51:00.786905Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6 namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: d788da9c-cd56-43aa-837b-aee97423de11 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6 to ip-10-0-134-20.ec2.internal metadata: creationTimestamp: "2026-07-01T14:51:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:51:00Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6.18be31fbfa0ba8f8 namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: 9ae9b1b3-cecc-4c22-8f84-1974883cad66 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-655cd54c75-lthr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6 namespace: kserve-ci-e2e-test resourceVersion: "15876" uid: d788da9c-cd56-43aa-837b-aee97423de11 kind: Event lastTimestamp: "2026-07-01T14:51:01Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:51:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:51:01Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6.18be31fc1397b496 namespace: kserve-ci-e2e-test resourceVersion: "15894" uid: e55e2b94-8751-41f5-9c92-c7e0cac32079 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6 namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: d788da9c-cd56-43aa-837b-aee97423de11 kind: Event lastTimestamp: "2026-07-01T14:51:01Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:32825628ffab44a0db6a86e1f74488e47dcab508ff646215bea8b81168e85657" already present on machine metadata: creationTimestamp: "2026-07-01T14:51:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:01Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6.18be31fc155a33ed namespace: kserve-ci-e2e-test resourceVersion: "15896" uid: 1ea08a1e-81c9-40f1-85d9-350ea1b7a7ed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6 namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: d788da9c-cd56-43aa-837b-aee97423de11 kind: Event lastTimestamp: "2026-07-01T14:51:01Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T14:51:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:01Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6.18be31fc1877e9ee namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: 7fa2eef9-a128-46a6-8623-b0d3f2f6aa81 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6 namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: d788da9c-cd56-43aa-837b-aee97423de11 kind: Event lastTimestamp: "2026-07-01T14:51:01Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T14:51:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:01Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6.18be31fc18de0517 namespace: kserve-ci-e2e-test resourceVersion: "15900" uid: 7fa00bf2-aaba-47dc-a3e4-ce1bef2ba544 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6 namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: d788da9c-cd56-43aa-837b-aee97423de11 kind: Event lastTimestamp: "2026-07-01T14:51:05Z" message: Container image "kserve/xgbserver:latest" already present on machine metadata: creationTimestamp: "2026-07-01T14:51:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:05Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6.18be31fd0fec9fb6 namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: 581435bc-edb0-4802-86ac-751b033b75ef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6 namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: d788da9c-cd56-43aa-837b-aee97423de11 kind: Event lastTimestamp: "2026-07-01T14:51:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T14:51:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:05Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6.18be31fd12e55905 namespace: kserve-ci-e2e-test resourceVersion: "15953" uid: ad16afb3-73a5-4e26-842d-5917daf9dec1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6 namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: d788da9c-cd56-43aa-837b-aee97423de11 kind: Event lastTimestamp: "2026-07-01T14:51:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T14:51:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:05Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6.18be31fd13a2e6ff namespace: kserve-ci-e2e-test resourceVersion: "15959" uid: 1f2c6a96-3bfb-4c90-9bb9-4f8e28085569 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6 namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: d788da9c-cd56-43aa-837b-aee97423de11 kind: Event lastTimestamp: "2026-07-01T14:51:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T14:51:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:05Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6.18be31fd13abc409 namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: b94d52c0-f743-4eee-9378-10fd8ca562da reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6 namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: d788da9c-cd56-43aa-837b-aee97423de11 kind: Event lastTimestamp: "2026-07-01T14:51:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T14:51:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:05Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6.18be31fd16441349 namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: a71399d7-6539-4061-9ca8-d13cacb373bd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6 namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: d788da9c-cd56-43aa-837b-aee97423de11 kind: Event lastTimestamp: "2026-07-01T14:51:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:51:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:05Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6.18be31fd16aee297 namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 8c870f7c-4d1e-4234-800c-83feb56375cb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T14:51:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6 namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: d788da9c-cd56-43aa-837b-aee97423de11 kind: Event lastTimestamp: "2026-07-01T14:52:52Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:51:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:52Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6.18be31fd4bcf81e0 namespace: kserve-ci-e2e-test resourceVersion: "16910" uid: 55d93757-8ca4-4cf2-ac01-6914b37dc7aa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6 namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: d788da9c-cd56-43aa-837b-aee97423de11 kind: Event lastTimestamp: "2026-07-01T14:52:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T14:52:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:51Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6.18be3215af26b37a namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: ee6f2c03-abed-461a-b62e-48d2e74519fd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6 namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: d788da9c-cd56-43aa-837b-aee97423de11 kind: Event lastTimestamp: "2026-07-01T14:52:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:52:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:51Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6.18be3215af2744e7 namespace: kserve-ci-e2e-test resourceVersion: "16891" uid: c016062c-b0e4-4999-a881-841187fd2c7f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6 namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: d788da9c-cd56-43aa-837b-aee97423de11 kind: Event lastTimestamp: "2026-07-01T14:52:52Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:52Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6.18be3215fa445a64 namespace: kserve-ci-e2e-test resourceVersion: "16907" uid: 6bb8c18a-366d-4c26-bea4-c9b7a164b619 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: 78382ab1-3466-41dc-8099-47c73bfe451f kind: Event lastTimestamp: "2026-07-01T14:51:00Z" message: 'Created pod: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c-s77m6' metadata: creationTimestamp: "2026-07-01T14:51:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:00Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c.18be31fbf95ea7bf namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: 363b920b-c3bb-4c5b-b30b-a6d293ba4284 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-raw-hpa-02737-predictor namespace: kserve-ci-e2e-test resourceVersion: "15833" uid: b371bc07-ffc5-4b00-88eb-a567e992547c kind: Event lastTimestamp: "2026-07-01T14:51:00Z" message: Scaled up replica set isvc-xgboost-graph-raw-hpa-02737-predictor-5c69dc797c from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:51:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:00Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor.18be31fbf4ba66c8 namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 4452e8a3-c8f0-45e7-95b9-25f8d3fb0aa5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:51:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-02737-predictor namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 0a34ff01-88b2-40a0-b241-ac53f10176e9 kind: Event lastTimestamp: "2026-07-01T14:51:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:51:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:45Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor.18be31ff73b2752c namespace: kserve-ci-e2e-test resourceVersion: "16280" uid: 09809cc9-4104-4204-8a0b-1a9ca65027b8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:51:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-02737-predictor namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 0a34ff01-88b2-40a0-b241-ac53f10176e9 kind: Event lastTimestamp: "2026-07-01T14:51:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:51:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:51:45Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor.18be31ff742c35e0 namespace: kserve-ci-e2e-test resourceVersion: "16281" uid: 14fb228e-f82f-42e3-986c-fd794aaee5a8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T14:51:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-02737-predictor namespace: kserve-ci-e2e-test resourceVersion: "16053" uid: 0a34ff01-88b2-40a0-b241-ac53f10176e9 kind: Event lastTimestamp: "2026-07-01T14:52:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:51:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:30Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor.18be3202f30c80cf namespace: kserve-ci-e2e-test resourceVersion: "16614" uid: fb075a51-9be6-49a0-b388-48cf7d1da3fe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T14:51:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-raw-hpa-02737-predictor namespace: kserve-ci-e2e-test resourceVersion: "16053" uid: 0a34ff01-88b2-40a0-b241-ac53f10176e9 kind: Event lastTimestamp: "2026-07-01T14:52:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:51:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:30Z" name: isvc-xgboost-graph-raw-hpa-02737-predictor.18be3202f3aaa28b namespace: kserve-ci-e2e-test resourceVersion: "16615" uid: 1cdca714-4f0e-4ed8-af38-d85499c4512a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-01T14:51:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-02737 namespace: kserve-ci-e2e-test resourceVersion: "15814" uid: 9ddc98f7-5085-4bb0-9eac-e09694e84d8b kind: Event lastTimestamp: "2026-07-01T14:52:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:51:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:52:12Z" name: isvc-xgboost-graph-raw-hpa-02737.18be31fbf53da942 namespace: kserve-ci-e2e-test resourceVersion: "16478" uid: cc9d5695-c5c9-4904-b4a5-5aac452a4f09 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph-raw-hpa-02737 namespace: kserve-ci-e2e-test resourceVersion: "16476" uid: 9ddc98f7-5085-4bb0-9eac-e09694e84d8b kind: Event lastTimestamp: "2026-07-01T14:52:12Z" message: InferenceService [isvc-xgboost-graph-raw-hpa-02737] is Ready metadata: creationTimestamp: "2026-07-01T14:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:52:12Z" name: isvc-xgboost-graph-raw-hpa-02737.18be320cb00a75e6 namespace: kserve-ci-e2e-test resourceVersion: "16477" uid: 36dd7f60-6baf-4f3b-967a-bd03cfaf8193 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:52:51.020221Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z namespace: kserve-ci-e2e-test resourceVersion: "16843" uid: 8fbc12fa-60d5-4fe9-83f3-bb345d08bc00 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z to ip-10-0-134-20.ec2.internal metadata: creationTimestamp: "2026-07-01T14:52:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:52:51Z" name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z.18be3215a47682ae namespace: kserve-ci-e2e-test resourceVersion: "16847" uid: b46f3065-7b78-4df5-b980-2ee6e6bc7fb2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-655cd54c75-lthr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:51Z" involvedObject: apiVersion: v1 kind: Pod name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z namespace: kserve-ci-e2e-test resourceVersion: "16854" uid: 8fbc12fa-60d5-4fe9-83f3-bb345d08bc00 kind: Event lastTimestamp: "2026-07-01T14:52:51Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:52:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:52:51Z" name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z.18be3215be01b84e namespace: kserve-ci-e2e-test resourceVersion: "16898" uid: b0537eb8-9471-46d7-91c2-272758534eb6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 8fbc12fa-60d5-4fe9-83f3-bb345d08bc00 kind: Event lastTimestamp: "2026-07-01T14:52:51Z" message: Pulling image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" metadata: creationTimestamp: "2026-07-01T14:52:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:51Z" name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z.18be3215bf9a54bb namespace: kserve-ci-e2e-test resourceVersion: "16900" uid: da503e15-d83f-44b3-8db2-2c8195287183 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 8fbc12fa-60d5-4fe9-83f3-bb345d08bc00 kind: Event lastTimestamp: "2026-07-01T14:52:52Z" message: 'Successfully pulled image "gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display" in 1.458s (1.458s including waiting). Image size: 14813193 bytes.' metadata: creationTimestamp: "2026-07-01T14:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:52Z" name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z.18be3216168e0791 namespace: kserve-ci-e2e-test resourceVersion: "16914" uid: 31301d3c-e9c6-4f47-9061-33393ea45a50 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 8fbc12fa-60d5-4fe9-83f3-bb345d08bc00 kind: Event lastTimestamp: "2026-07-01T14:52:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T14:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:52Z" name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z.18be3216198f5cef namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: bc79ce50-5f2d-4f6c-a29c-5184ba23b6a4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 8fbc12fa-60d5-4fe9-83f3-bb345d08bc00 kind: Event lastTimestamp: "2026-07-01T14:52:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T14:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:52Z" name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z.18be32161a4e3fd1 namespace: kserve-ci-e2e-test resourceVersion: "16916" uid: 0870b348-fb8d-41b8-bbee-fbc914a71eec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 8fbc12fa-60d5-4fe9-83f3-bb345d08bc00 kind: Event lastTimestamp: "2026-07-01T14:52:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T14:52:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:53Z" name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z.18be32161a5940c5 namespace: kserve-ci-e2e-test resourceVersion: "16917" uid: e742b4f0-f33a-49a0-9bb2-1789f267e3c0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 8fbc12fa-60d5-4fe9-83f3-bb345d08bc00 kind: Event lastTimestamp: "2026-07-01T14:52:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T14:52:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:53Z" name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z.18be32161d46b379 namespace: kserve-ci-e2e-test resourceVersion: "16918" uid: 38c8bade-7170-456d-877a-73bcbd5287e6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 8fbc12fa-60d5-4fe9-83f3-bb345d08bc00 kind: Event lastTimestamp: "2026-07-01T14:52:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:52:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:53Z" name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z.18be32161dbf0c01 namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 68ed6f75-ab54-48f5-88cf-9a404c125567 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:54:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 8fbc12fa-60d5-4fe9-83f3-bb345d08bc00 kind: Event lastTimestamp: "2026-07-01T14:54:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T14:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:54:46Z" name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z.18be32307f97b641 namespace: kserve-ci-e2e-test resourceVersion: "17893" uid: f0a8a13b-5a59-4fe3-bad9-026714fd95b2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:54:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 8fbc12fa-60d5-4fe9-83f3-bb345d08bc00 kind: Event lastTimestamp: "2026-07-01T14:54:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:54:46Z" name: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z.18be32307f982420 namespace: kserve-ci-e2e-test resourceVersion: "17894" uid: 78a976cb-3f1a-4077-a0ea-b7812049b482 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: message-dumper-raw-4ce12-predictor-9d84f76c9 namespace: kserve-ci-e2e-test resourceVersion: "16831" uid: cdd1f793-3586-47ea-8804-24f52c8311fd kind: Event lastTimestamp: "2026-07-01T14:52:51Z" message: 'Created pod: message-dumper-raw-4ce12-predictor-9d84f76c9-4r95z' metadata: creationTimestamp: "2026-07-01T14:52:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:51Z" name: message-dumper-raw-4ce12-predictor-9d84f76c9.18be3215a3eb95c3 namespace: kserve-ci-e2e-test resourceVersion: "16845" uid: cdabcbda-4c16-4bfb-9a20-3f77d888ff3a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: message-dumper-raw-4ce12-predictor namespace: kserve-ci-e2e-test resourceVersion: "16829" uid: bc29b12f-6194-41c0-9403-be8a28a8aac4 kind: Event lastTimestamp: "2026-07-01T14:52:50Z" message: Scaled up replica set message-dumper-raw-4ce12-predictor-9d84f76c9 from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:52:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:50Z" name: message-dumper-raw-4ce12-predictor.18be32159dc5efff namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 683e7082-c878-4048-a072-a957c90c1b0a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:53:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-4ce12-predictor namespace: kserve-ci-e2e-test resourceVersion: "16830" uid: 6c44816b-e9e5-45d9-826d-aff17d53d3f9 kind: Event lastTimestamp: "2026-07-01T14:53:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:53:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:53:36Z" name: message-dumper-raw-4ce12-predictor.18be32191dd907ed namespace: kserve-ci-e2e-test resourceVersion: "17318" uid: af6f8d64-af6b-4991-ab46-1931b87cb165 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:53:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-4ce12-predictor namespace: kserve-ci-e2e-test resourceVersion: "16830" uid: 6c44816b-e9e5-45d9-826d-aff17d53d3f9 kind: Event lastTimestamp: "2026-07-01T14:53:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:53:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:53:36Z" name: message-dumper-raw-4ce12-predictor.18be32191e87663c namespace: kserve-ci-e2e-test resourceVersion: "17320" uid: 71153d2b-9b19-4db2-967a-1c34c4558f6d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:53:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-4ce12-predictor namespace: kserve-ci-e2e-test resourceVersion: "17030" uid: 6c44816b-e9e5-45d9-826d-aff17d53d3f9 kind: Event lastTimestamp: "2026-07-01T14:53:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:53:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:53:20Z" name: message-dumper-raw-4ce12-predictor.18be321c9e1dc76d namespace: kserve-ci-e2e-test resourceVersion: "17202" uid: 9dd463c2-a59d-436a-9cf3-0b6d4cd024ec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:53:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: message-dumper-raw-4ce12-predictor namespace: kserve-ci-e2e-test resourceVersion: "17030" uid: 6c44816b-e9e5-45d9-826d-aff17d53d3f9 kind: Event lastTimestamp: "2026-07-01T14:53:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T14:53:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:53:20Z" name: message-dumper-raw-4ce12-predictor.18be321c9eb2aee7 namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: d31a113f-cfab-4526-8382-81dbdca59e52 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T14:52:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-4ce12 namespace: kserve-ci-e2e-test resourceVersion: "16822" uid: 905a72b4-59d9-4d3f-9b3f-a26ffcdd9637 kind: Event lastTimestamp: "2026-07-01T14:53:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:52:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:53:01Z" name: message-dumper-raw-4ce12.18be32159e1983a6 namespace: kserve-ci-e2e-test resourceVersion: "16994" uid: f0177ee4-0b86-4048-aa37-66d9ea5af8c6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-4ce12 namespace: kserve-ci-e2e-test resourceVersion: "16822" uid: 905a72b4-59d9-4d3f-9b3f-a26ffcdd9637 kind: Event lastTimestamp: "2026-07-01T14:52:50Z" message: 'Failed to update status for InferenceService "message-dumper-raw-4ce12": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "message-dumper-raw-4ce12": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:52:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:52:50Z" name: message-dumper-raw-4ce12.18be32159f57ac06 namespace: kserve-ci-e2e-test resourceVersion: "16838" uid: 009b573c-6c79-43b9-8394-0dcbdbb0db0d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:53:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: message-dumper-raw-4ce12 namespace: kserve-ci-e2e-test resourceVersion: "16990" uid: 905a72b4-59d9-4d3f-9b3f-a26ffcdd9637 kind: Event lastTimestamp: "2026-07-01T14:53:01Z" message: InferenceService [message-dumper-raw-4ce12] is Ready metadata: creationTimestamp: "2026-07-01T14:53:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:53:01Z" name: message-dumper-raw-4ce12.18be3217f9a178ac namespace: kserve-ci-e2e-test resourceVersion: "16992" uid: 27f4230c-eab8-486d-b1ae-e353fb0a7370 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:50:50.238529Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-63a63-6dc96485f-65cxv namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: a6c095aa-9623-4d2b-9fc7-a9d49c3081ff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-63a63-6dc96485f-65cxv to ip-10-0-134-63.ec2.internal metadata: creationTimestamp: "2026-07-01T14:50:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:50:50Z" name: model-chainer-raw-63a63-6dc96485f-65cxv.18be31f98550267f namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: a428afed-16b0-41d4-96c0-5fe1c9e6052e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-655cd54c75-lthr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:50Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-63a63-6dc96485f-65cxv namespace: kserve-ci-e2e-test resourceVersion: "15668" uid: a6c095aa-9623-4d2b-9fc7-a9d49c3081ff kind: Event lastTimestamp: "2026-07-01T14:50:50Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:50:50Z" name: model-chainer-raw-63a63-6dc96485f-65cxv.18be31f99ee150ca namespace: kserve-ci-e2e-test resourceVersion: "15677" uid: 4b1079a1-243e-4fb4-a844-7e283d58696f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-63a63} kind: Pod name: model-chainer-raw-63a63-6dc96485f-65cxv namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: a6c095aa-9623-4d2b-9fc7-a9d49c3081ff kind: Event lastTimestamp: "2026-07-01T14:50:50Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:faf8bd9583948cd226ae9950de299cfff5a107a2e66036dae859f635a11abe1a" metadata: creationTimestamp: "2026-07-01T14:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:50Z" name: model-chainer-raw-63a63-6dc96485f-65cxv.18be31f9a07c6c26 namespace: kserve-ci-e2e-test resourceVersion: "15679" uid: bc863768-858c-486b-979c-96ea81f873ba reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-63.ec2.internal source: component: kubelet host: ip-10-0-134-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-63a63} kind: Pod name: model-chainer-raw-63a63-6dc96485f-65cxv namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: a6c095aa-9623-4d2b-9fc7-a9d49c3081ff kind: Event lastTimestamp: "2026-07-01T14:50:53Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:faf8bd9583948cd226ae9950de299cfff5a107a2e66036dae859f635a11abe1a" in 2.662s (2.662s including waiting). Image size: 218070141 bytes.' metadata: creationTimestamp: "2026-07-01T14:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:53Z" name: model-chainer-raw-63a63-6dc96485f-65cxv.18be31fa3f2c9a34 namespace: kserve-ci-e2e-test resourceVersion: "15700" uid: ccf7f396-60a6-461b-9dc7-bcbb189ff11f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-63.ec2.internal source: component: kubelet host: ip-10-0-134-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-63a63} kind: Pod name: model-chainer-raw-63a63-6dc96485f-65cxv namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: a6c095aa-9623-4d2b-9fc7-a9d49c3081ff kind: Event lastTimestamp: "2026-07-01T14:50:53Z" message: 'Created container: model-chainer-raw-63a63' metadata: creationTimestamp: "2026-07-01T14:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:53Z" name: model-chainer-raw-63a63-6dc96485f-65cxv.18be31fa4230290c namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: b8971c65-0f38-41a3-b95b-88b2454ef221 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-63.ec2.internal source: component: kubelet host: ip-10-0-134-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-63a63} kind: Pod name: model-chainer-raw-63a63-6dc96485f-65cxv namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: a6c095aa-9623-4d2b-9fc7-a9d49c3081ff kind: Event lastTimestamp: "2026-07-01T14:50:53Z" message: Started container model-chainer-raw-63a63 metadata: creationTimestamp: "2026-07-01T14:50:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:50:53Z" name: model-chainer-raw-63a63-6dc96485f-65cxv.18be31fa42a1defa namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: 9d9f4193-7ea4-442a-b419-5ddc78564bc7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-63.ec2.internal source: component: kubelet host: ip-10-0-134-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:51:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-63a63} kind: Pod name: model-chainer-raw-63a63-6dc96485f-65cxv namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: a6c095aa-9623-4d2b-9fc7-a9d49c3081ff kind: Event lastTimestamp: "2026-07-01T14:51:00Z" message: Stopping container model-chainer-raw-63a63 metadata: creationTimestamp: "2026-07-01T14:51:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:00Z" name: model-chainer-raw-63a63-6dc96485f-65cxv.18be31fbddf0e4ef namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: af3ac9f2-8f66-4151-8152-00e714345f31 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-63.ec2.internal source: component: kubelet host: ip-10-0-134-63.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-01T14:51:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-63a63} kind: Pod name: model-chainer-raw-63a63-6dc96485f-65cxv namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: a6c095aa-9623-4d2b-9fc7-a9d49c3081ff kind: Event lastTimestamp: "2026-07-01T14:51:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T14:51:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:51:30Z" name: model-chainer-raw-63a63-6dc96485f-65cxv.18be31fbde3bd10b namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: 976f8f7a-bcd3-4575-822d-80d9ff3c361c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-63.ec2.internal source: component: kubelet host: ip-10-0-134-63.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-63a63-6dc96485f namespace: kserve-ci-e2e-test resourceVersion: "15656" uid: c7047a14-808b-44dd-b664-6fd19d61c7d6 kind: Event lastTimestamp: "2026-07-01T14:50:50Z" message: 'Created pod: model-chainer-raw-63a63-6dc96485f-65cxv' metadata: creationTimestamp: "2026-07-01T14:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:50Z" name: model-chainer-raw-63a63-6dc96485f.18be31f984ae1403 namespace: kserve-ci-e2e-test resourceVersion: "15661" uid: 22270fa1-d02b-4455-a4ae-8fd6d7c38cc8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-63a63 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: 1eb53a79-ac37-4123-a517-09454cae5499 kind: Event lastTimestamp: "2026-07-01T14:50:50Z" message: Scaled up replica set model-chainer-raw-63a63-6dc96485f from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:50:50Z" name: model-chainer-raw-63a63.18be31f9836d594a namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: 93934eb7-a449-4ba1-9530-5fd12acea4ca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:50:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-63a63 namespace: kserve-ci-e2e-test resourceVersion: "15670" uid: 18ab22f2-bca9-4302-92c7-4b85d4cf52f4 kind: Event lastTimestamp: "2026-07-01T14:50:50Z" message: InferenceGraph [model-chainer-raw-63a63] is Ready metadata: creationTimestamp: "2026-07-01T14:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:50:50Z" name: model-chainer-raw-63a63.18be31f989e877a9 namespace: kserve-ci-e2e-test resourceVersion: "15672" uid: 7a114263-9aae-4ecc-b136-344dc2a59280 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:52:40.630624Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-02737-5f44fc9b8f-k59lz namespace: kserve-ci-e2e-test resourceVersion: "16702" uid: 34e1724c-9be2-4ae2-b52c-1a59d71fa54a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-raw-hpa-02737-5f44fc9b8f-k59lz to ip-10-0-134-63.ec2.internal metadata: creationTimestamp: "2026-07-01T14:52:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:52:40Z" name: model-chainer-raw-hpa-02737-5f44fc9b8f-k59lz.18be32133931dbca namespace: kserve-ci-e2e-test resourceVersion: "16706" uid: 93f14d0e-dd91-4502-b748-341bca1a8d04 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-655cd54c75-lthr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:40Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-02737-5f44fc9b8f-k59lz namespace: kserve-ci-e2e-test resourceVersion: "16704" uid: 34e1724c-9be2-4ae2-b52c-1a59d71fa54a kind: Event lastTimestamp: "2026-07-01T14:52:40Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-raw-hpa-02737-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T14:52:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:40Z" name: model-chainer-raw-hpa-02737-5f44fc9b8f-k59lz.18be321345403d67 namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: ef46ef91-9286-46d9-9381-668c76e93637 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-63.ec2.internal source: component: kubelet host: ip-10-0-134-63.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:41Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-raw-hpa-02737-5f44fc9b8f-k59lz namespace: kserve-ci-e2e-test resourceVersion: "16710" uid: 34e1724c-9be2-4ae2-b52c-1a59d71fa54a kind: Event lastTimestamp: "2026-07-01T14:52:41Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:52:41Z" name: model-chainer-raw-hpa-02737-5f44fc9b8f-k59lz.18be321376b672a4 namespace: kserve-ci-e2e-test resourceVersion: "16726" uid: 0e7eda28-c55b-4950-bb77-d5181fe03559 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-02737} kind: Pod name: model-chainer-raw-hpa-02737-5f44fc9b8f-k59lz namespace: kserve-ci-e2e-test resourceVersion: "16704" uid: 34e1724c-9be2-4ae2-b52c-1a59d71fa54a kind: Event lastTimestamp: "2026-07-01T14:52:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:faf8bd9583948cd226ae9950de299cfff5a107a2e66036dae859f635a11abe1a" already present on machine metadata: creationTimestamp: "2026-07-01T14:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:41Z" name: model-chainer-raw-hpa-02737-5f44fc9b8f-k59lz.18be32137925415f namespace: kserve-ci-e2e-test resourceVersion: "16729" uid: 0b851f52-dd23-4452-913f-92669b734fb7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-63.ec2.internal source: component: kubelet host: ip-10-0-134-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-02737} kind: Pod name: model-chainer-raw-hpa-02737-5f44fc9b8f-k59lz namespace: kserve-ci-e2e-test resourceVersion: "16704" uid: 34e1724c-9be2-4ae2-b52c-1a59d71fa54a kind: Event lastTimestamp: "2026-07-01T14:52:41Z" message: 'Created container: model-chainer-raw-hpa-02737' metadata: creationTimestamp: "2026-07-01T14:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:41Z" name: model-chainer-raw-hpa-02737-5f44fc9b8f-k59lz.18be32137d1b61a4 namespace: kserve-ci-e2e-test resourceVersion: "16731" uid: 087c27e6-c29b-47e9-8c61-7fca93ee7528 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-63.ec2.internal source: component: kubelet host: ip-10-0-134-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-02737} kind: Pod name: model-chainer-raw-hpa-02737-5f44fc9b8f-k59lz namespace: kserve-ci-e2e-test resourceVersion: "16704" uid: 34e1724c-9be2-4ae2-b52c-1a59d71fa54a kind: Event lastTimestamp: "2026-07-01T14:52:41Z" message: Started container model-chainer-raw-hpa-02737 metadata: creationTimestamp: "2026-07-01T14:52:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:41Z" name: model-chainer-raw-hpa-02737-5f44fc9b8f-k59lz.18be32137d96dd03 namespace: kserve-ci-e2e-test resourceVersion: "16732" uid: 84437029-0fbe-48c4-b2bd-0c12201adad0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-63.ec2.internal source: component: kubelet host: ip-10-0-134-63.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-02737} kind: Pod name: model-chainer-raw-hpa-02737-5f44fc9b8f-k59lz namespace: kserve-ci-e2e-test resourceVersion: "16704" uid: 34e1724c-9be2-4ae2-b52c-1a59d71fa54a kind: Event lastTimestamp: "2026-07-01T14:52:50Z" message: Stopping container model-chainer-raw-hpa-02737 metadata: creationTimestamp: "2026-07-01T14:52:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:52:50Z" name: model-chainer-raw-hpa-02737-5f44fc9b8f-k59lz.18be32158f05cf3a namespace: kserve-ci-e2e-test resourceVersion: "16817" uid: 3149efdd-431a-4df1-96ac-d23a347c0a39 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-63.ec2.internal source: component: kubelet host: ip-10-0-134-63.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-01T14:52:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer-raw-hpa-02737} kind: Pod name: model-chainer-raw-hpa-02737-5f44fc9b8f-k59lz namespace: kserve-ci-e2e-test resourceVersion: "16704" uid: 34e1724c-9be2-4ae2-b52c-1a59d71fa54a kind: Event lastTimestamp: "2026-07-01T14:53:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-01T14:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:53:17Z" name: model-chainer-raw-hpa-02737-5f44fc9b8f-k59lz.18be32160f19495f namespace: kserve-ci-e2e-test resourceVersion: "17174" uid: 133c1f29-0f0c-4ef8-98f7-89eec18d1347 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-63.ec2.internal source: component: kubelet host: ip-10-0-134-63.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-raw-hpa-02737-5f44fc9b8f namespace: kserve-ci-e2e-test resourceVersion: "16698" uid: 77bb83ba-2c90-4f45-928f-b2a683bf918e kind: Event lastTimestamp: "2026-07-01T14:52:40Z" message: 'Created pod: model-chainer-raw-hpa-02737-5f44fc9b8f-k59lz' metadata: creationTimestamp: "2026-07-01T14:52:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:40Z" name: model-chainer-raw-hpa-02737-5f44fc9b8f.18be321338a4c1fc namespace: kserve-ci-e2e-test resourceVersion: "16703" uid: c80b88f5-56f2-4952-bbed-dd2ad7c73987 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer-raw-hpa-02737 namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: 655ed399-6e90-4f72-9474-28566aa5da2a kind: Event lastTimestamp: "2026-07-01T14:52:40Z" message: Scaled up replica set model-chainer-raw-hpa-02737-5f44fc9b8f from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:52:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:52:40Z" name: model-chainer-raw-hpa-02737.18be321336d42570 namespace: kserve-ci-e2e-test resourceVersion: "16700" uid: a5390a27-90cb-4955-9d93-edd269e89d2d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:52:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer-raw-hpa-02737 namespace: kserve-ci-e2e-test resourceVersion: "16712" uid: 9dbb5011-b2c4-4562-9848-18e3f377940d kind: Event lastTimestamp: "2026-07-01T14:52:40Z" message: InferenceGraph [model-chainer-raw-hpa-02737] is Ready metadata: creationTimestamp: "2026-07-01T14:52:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:52:40Z" name: model-chainer-raw-hpa-02737.18be32133e52011c namespace: kserve-ci-e2e-test resourceVersion: "16713" uid: c8765400-c95b-4e41-9fcd-352202423356 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T14:59:33.188747Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j namespace: kserve-ci-e2e-test resourceVersion: "20257" uid: e83134e2-d1de-4a5a-95ca-f1441788fc7f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-fd58b-predictor-55b484874b-pdv4j to ip-10-0-134-20.ec2.internal metadata: creationTimestamp: "2026-07-01T14:59:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T14:59:33Z" name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j.18be327347934bb4 namespace: kserve-ci-e2e-test resourceVersion: "20263" uid: 01147c9e-d9ff-4ebf-a55f-159288e4d28a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-655cd54c75-lthr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:33Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j namespace: kserve-ci-e2e-test resourceVersion: "20267" uid: e83134e2-d1de-4a5a-95ca-f1441788fc7f kind: Event lastTimestamp: "2026-07-01T14:59:33Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T14:59:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T14:59:33Z" name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j.18be327360e9f2ef namespace: kserve-ci-e2e-test resourceVersion: "20277" uid: 64001692-4b95-4dce-9cbc-c2b8c346a44b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j namespace: kserve-ci-e2e-test resourceVersion: "20259" uid: e83134e2-d1de-4a5a-95ca-f1441788fc7f kind: Event lastTimestamp: "2026-07-01T14:59:33Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:32825628ffab44a0db6a86e1f74488e47dcab508ff646215bea8b81168e85657" already present on machine metadata: creationTimestamp: "2026-07-01T14:59:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:33Z" name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j.18be327362ea605f namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: 7ffddd79-fef7-42d9-a285-5a4bfd41f2c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j namespace: kserve-ci-e2e-test resourceVersion: "20259" uid: e83134e2-d1de-4a5a-95ca-f1441788fc7f kind: Event lastTimestamp: "2026-07-01T14:59:33Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T14:59:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:33Z" name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j.18be327365b912fa namespace: kserve-ci-e2e-test resourceVersion: "20280" uid: d6b7a8df-c3c9-4d92-819a-f19ab363cdbc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:33Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j namespace: kserve-ci-e2e-test resourceVersion: "20259" uid: e83134e2-d1de-4a5a-95ca-f1441788fc7f kind: Event lastTimestamp: "2026-07-01T14:59:33Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T14:59:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:33Z" name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j.18be327366255086 namespace: kserve-ci-e2e-test resourceVersion: "20281" uid: 75dbdeb8-a1d4-479a-a1eb-7f28f45387f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j namespace: kserve-ci-e2e-test resourceVersion: "20259" uid: e83134e2-d1de-4a5a-95ca-f1441788fc7f kind: Event lastTimestamp: "2026-07-01T14:59:37Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T14:59:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:37Z" name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j.18be327452ad0862 namespace: kserve-ci-e2e-test resourceVersion: "20316" uid: 613f9fab-d98e-4011-a37f-754b3be4198b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j namespace: kserve-ci-e2e-test resourceVersion: "20259" uid: e83134e2-d1de-4a5a-95ca-f1441788fc7f kind: Event lastTimestamp: "2026-07-01T14:59:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T14:59:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:37Z" name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j.18be327455d49916 namespace: kserve-ci-e2e-test resourceVersion: "20319" uid: 3d15dba4-1e09-456e-afd2-2b02de5cbbd9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j namespace: kserve-ci-e2e-test resourceVersion: "20259" uid: e83134e2-d1de-4a5a-95ca-f1441788fc7f kind: Event lastTimestamp: "2026-07-01T14:59:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T14:59:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:37Z" name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j.18be3274566479ba namespace: kserve-ci-e2e-test resourceVersion: "20320" uid: 6a1deb34-5211-48b2-82d5-a23d2b1da5ca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j namespace: kserve-ci-e2e-test resourceVersion: "20259" uid: e83134e2-d1de-4a5a-95ca-f1441788fc7f kind: Event lastTimestamp: "2026-07-01T14:59:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T14:59:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:37Z" name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j.18be3274566efb6a namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: 4519e358-b599-44a3-81fb-c8f62ae3c5b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j namespace: kserve-ci-e2e-test resourceVersion: "20259" uid: e83134e2-d1de-4a5a-95ca-f1441788fc7f kind: Event lastTimestamp: "2026-07-01T14:59:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T14:59:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:37Z" name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j.18be32745a645ba7 namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: fef0167d-75a4-4b2c-8e1f-84ae309ad8cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j namespace: kserve-ci-e2e-test resourceVersion: "20259" uid: e83134e2-d1de-4a5a-95ca-f1441788fc7f kind: Event lastTimestamp: "2026-07-01T14:59:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T14:59:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T14:59:37Z" name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j.18be32745ad552d6 namespace: kserve-ci-e2e-test resourceVersion: "20326" uid: 15412e64-0ac0-4f05-988f-f36741817802 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T14:59:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j namespace: kserve-ci-e2e-test resourceVersion: "20259" uid: e83134e2-d1de-4a5a-95ca-f1441788fc7f kind: Event lastTimestamp: "2026-07-01T15:01:05Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T14:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:05Z" name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j.18be3274ca938274 namespace: kserve-ci-e2e-test resourceVersion: "20981" uid: ff1828b0-8166-441d-9f00-cb35033d99dc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j namespace: kserve-ci-e2e-test resourceVersion: "20259" uid: e83134e2-d1de-4a5a-95ca-f1441788fc7f kind: Event lastTimestamp: "2026-07-01T15:01:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T15:01:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:03Z" name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j.18be328847d2321e namespace: kserve-ci-e2e-test resourceVersion: "20963" uid: ae054d8a-8199-45f3-bfe4-54abb7cb18c5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j namespace: kserve-ci-e2e-test resourceVersion: "20259" uid: e83134e2-d1de-4a5a-95ca-f1441788fc7f kind: Event lastTimestamp: "2026-07-01T15:01:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:01:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:03Z" name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j.18be328847d2f00e namespace: kserve-ci-e2e-test resourceVersion: "20964" uid: 1196c312-77a0-4a0b-a033-6779b72dd3a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j namespace: kserve-ci-e2e-test resourceVersion: "20259" uid: e83134e2-d1de-4a5a-95ca-f1441788fc7f kind: Event lastTimestamp: "2026-07-01T15:01:05Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:01:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:05Z" name: raw-sklearn-fd58b-predictor-55b484874b-pdv4j.18be3288d0cb4928 namespace: kserve-ci-e2e-test resourceVersion: "20980" uid: bdb6eecb-f73a-41df-8277-37deb7828c20 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-fd58b-predictor-55b484874b namespace: kserve-ci-e2e-test resourceVersion: "20248" uid: 2bece199-9791-4cd3-be08-b093a79806f5 kind: Event lastTimestamp: "2026-07-01T14:59:33Z" message: 'Created pod: raw-sklearn-fd58b-predictor-55b484874b-pdv4j' metadata: creationTimestamp: "2026-07-01T14:59:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:59:33Z" name: raw-sklearn-fd58b-predictor-55b484874b.18be327347163b17 namespace: kserve-ci-e2e-test resourceVersion: "20258" uid: 68d04cf7-56b1-46ab-ab58-85b1304fdb66 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-fd58b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20246" uid: 92486cc2-46ea-4e03-8025-d3cd7df71683 kind: Event lastTimestamp: "2026-07-01T14:59:33Z" message: Scaled up replica set raw-sklearn-fd58b-predictor-55b484874b from 0 to 1 metadata: creationTimestamp: "2026-07-01T14:59:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T14:59:33Z" name: raw-sklearn-fd58b-predictor.18be32734391356a namespace: kserve-ci-e2e-test resourceVersion: "20250" uid: 0898d391-91fe-4666-a09c-2ef4a629e20b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:59:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-fd58b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: 64935712-f780-4996-abe4-d33588ef315e kind: Event lastTimestamp: "2026-07-01T15:00:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:59:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:00:18Z" name: raw-sklearn-fd58b-predictor.18be3276c23c445c namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 66c8c3ec-72fc-4595-b04b-332966161fe3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T14:59:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-fd58b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: 64935712-f780-4996-abe4-d33588ef315e kind: Event lastTimestamp: "2026-07-01T15:00:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T14:59:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:00:18Z" name: raw-sklearn-fd58b-predictor.18be3276c2b0d9f8 namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: 0826fe4b-9e9a-45b2-831e-5528975c9438 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T15:00:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-fd58b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20404" uid: 64935712-f780-4996-abe4-d33588ef315e kind: Event lastTimestamp: "2026-07-01T15:01:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:00:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:03Z" name: raw-sklearn-fd58b-predictor.18be327a4314c77b namespace: kserve-ci-e2e-test resourceVersion: "20946" uid: 93327f37-5fb8-4bef-9716-9fe6ab3eb660 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T15:00:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-fd58b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20404" uid: 64935712-f780-4996-abe4-d33588ef315e kind: Event lastTimestamp: "2026-07-01T15:01:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:00:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:03Z" name: raw-sklearn-fd58b-predictor.18be327a43938a5a namespace: kserve-ci-e2e-test resourceVersion: "20947" uid: fad97180-e788-452f-a637-4f5fd9633c24 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-01T14:59:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-fd58b namespace: kserve-ci-e2e-test resourceVersion: "20239" uid: 2e3f6f09-577e-45e6-9733-3719e6e98195 kind: Event lastTimestamp: "2026-07-01T15:00:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T14:59:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:00:55Z" name: raw-sklearn-fd58b.18be327343a9c71b namespace: kserve-ci-e2e-test resourceVersion: "20889" uid: ccd77691-6d21-4e66-96ac-86a0e7311823 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T14:59:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-fd58b namespace: kserve-ci-e2e-test resourceVersion: "20239" uid: 2e3f6f09-577e-45e6-9733-3719e6e98195 kind: Event lastTimestamp: "2026-07-01T14:59:33Z" message: 'Failed to update status for InferenceService "raw-sklearn-fd58b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-fd58b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T14:59:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T14:59:33Z" name: raw-sklearn-fd58b.18be32734427cb26 namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: 1f5fc5f6-3f65-4680-8e9a-cefdb7625cff reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:00:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-fd58b namespace: kserve-ci-e2e-test resourceVersion: "20887" uid: 2e3f6f09-577e-45e6-9733-3719e6e98195 kind: Event lastTimestamp: "2026-07-01T15:00:55Z" message: InferenceService [raw-sklearn-fd58b] is Ready metadata: creationTimestamp: "2026-07-01T15:00:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:00:55Z" name: raw-sklearn-fd58b.18be328683ac13d5 namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: a202ed4c-698c-44ff-9715-a39f47bae3d4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-01T15:01:08.494319Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9 namespace: kserve-ci-e2e-test resourceVersion: "21021" uid: 86300943-7406-4dd0-b543-f96086127c11 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9 to ip-10-0-134-20.ec2.internal metadata: creationTimestamp: "2026-07-01T15:01:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-01T15:01:08Z" name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9.18be3289783b0c09 namespace: kserve-ci-e2e-test resourceVersion: "21025" uid: c381c88d-e569-45d8-a873-7775baab4ead reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-655cd54c75-lthr8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:08Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 86300943-7406-4dd0-b543-f96086127c11 kind: Event lastTimestamp: "2026-07-01T15:01:08Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "raw-sklearn-runtime-59568-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-01T15:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:08Z" name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9.18be328982ab48f0 namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: c285b16e-3831-437e-8ade-73b24b04659e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:09Z" involvedObject: apiVersion: v1 kind: Pod name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9 namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: 86300943-7406-4dd0-b543-f96086127c11 kind: Event lastTimestamp: "2026-07-01T15:01:09Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-01T15:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-01T15:01:09Z" name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9.18be3289b52c2e91 namespace: kserve-ci-e2e-test resourceVersion: "21050" uid: ae9071b2-91e4-4832-8ff7-f36d74aff86e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 86300943-7406-4dd0-b543-f96086127c11 kind: Event lastTimestamp: "2026-07-01T15:01:09Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:32825628ffab44a0db6a86e1f74488e47dcab508ff646215bea8b81168e85657" already present on machine metadata: creationTimestamp: "2026-07-01T15:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:09Z" name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9.18be3289b6cd91c5 namespace: kserve-ci-e2e-test resourceVersion: "21052" uid: 33b22a82-2602-407b-b4ef-b6ac1936d018 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 86300943-7406-4dd0-b543-f96086127c11 kind: Event lastTimestamp: "2026-07-01T15:01:09Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-01T15:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:09Z" name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9.18be3289bae290a5 namespace: kserve-ci-e2e-test resourceVersion: "21053" uid: f739319a-aab5-456f-a744-1d19dae9c0e3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:09Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 86300943-7406-4dd0-b543-f96086127c11 kind: Event lastTimestamp: "2026-07-01T15:01:09Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-01T15:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:09Z" name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9.18be3289bb548d86 namespace: kserve-ci-e2e-test resourceVersion: "21054" uid: 2ee6fa24-fb19-482a-a6e2-81a82440f2fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 86300943-7406-4dd0-b543-f96086127c11 kind: Event lastTimestamp: "2026-07-01T15:01:14Z" message: Container image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1690" already present on machine metadata: creationTimestamp: "2026-07-01T15:01:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:14Z" name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9.18be328ac4c18032 namespace: kserve-ci-e2e-test resourceVersion: "21095" uid: c68f3681-f6ba-4d0e-9a89-1c5cf7c41c34 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 86300943-7406-4dd0-b543-f96086127c11 kind: Event lastTimestamp: "2026-07-01T15:01:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-01T15:01:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:14Z" name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9.18be328ac7b58a3b namespace: kserve-ci-e2e-test resourceVersion: "21098" uid: 55933983-75fc-4206-a968-5f9da1812558 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 86300943-7406-4dd0-b543-f96086127c11 kind: Event lastTimestamp: "2026-07-01T15:01:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-01T15:01:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:14Z" name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9.18be328ac81d2e2c namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: 00bd20e2-18b5-466d-887f-d6c12a091e23 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 86300943-7406-4dd0-b543-f96086127c11 kind: Event lastTimestamp: "2026-07-01T15:01:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-01T15:01:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:14Z" name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9.18be328ac8261a25 namespace: kserve-ci-e2e-test resourceVersion: "21100" uid: 7233bfa9-ca79-49e1-9663-c6c826fea632 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 86300943-7406-4dd0-b543-f96086127c11 kind: Event lastTimestamp: "2026-07-01T15:01:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-01T15:01:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:14Z" name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9.18be328acb54eb51 namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: e2438690-edaf-47a7-ac70-94d53c14c4bf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 86300943-7406-4dd0-b543-f96086127c11 kind: Event lastTimestamp: "2026-07-01T15:01:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:01:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:01:14Z" name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9.18be328acbca0ed0 namespace: kserve-ci-e2e-test resourceVersion: "21104" uid: 63749eb2-f0a0-43a4-842d-a9b67f83b85b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-01T15:01:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 86300943-7406-4dd0-b543-f96086127c11 kind: Event lastTimestamp: "2026-07-01T15:02:42Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:01:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:42Z" name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9.18be328b3ca3e345 namespace: kserve-ci-e2e-test resourceVersion: "21752" uid: af222d4a-ffc7-4413-928b-5578ccc7bd4b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 86300943-7406-4dd0-b543-f96086127c11 kind: Event lastTimestamp: "2026-07-01T15:02:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-01T15:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:38Z" name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9.18be329e74c1ca20 namespace: kserve-ci-e2e-test resourceVersion: "21720" uid: 681f93ac-7f4b-4fed-a53e-b9c4d51e86f5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 86300943-7406-4dd0-b543-f96086127c11 kind: Event lastTimestamp: "2026-07-01T15:02:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-01T15:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:38Z" name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9.18be329e74c2e674 namespace: kserve-ci-e2e-test resourceVersion: "21721" uid: 1577bb69-d657-4203-bad0-37f0cc3721f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9 namespace: kserve-ci-e2e-test resourceVersion: "21023" uid: 86300943-7406-4dd0-b543-f96086127c11 kind: Event lastTimestamp: "2026-07-01T15:02:42Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-01T15:02:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-01T15:02:42Z" name: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9.18be329f42e51b22 namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: f18bcd85-ef9a-405d-9a4c-f668a7d2f534 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-20.ec2.internal source: component: kubelet host: ip-10-0-134-20.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: raw-sklearn-runtime-59568-predictor-6849b9cc8b namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: 7ac27463-8210-41a8-86a3-9effb19939e8 kind: Event lastTimestamp: "2026-07-01T15:01:08Z" message: 'Created pod: raw-sklearn-runtime-59568-predictor-6849b9cc8b-7crw9' metadata: creationTimestamp: "2026-07-01T15:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:08Z" name: raw-sklearn-runtime-59568-predictor-6849b9cc8b.18be32897796dacb namespace: kserve-ci-e2e-test resourceVersion: "21022" uid: efb0cdd2-2944-4c93-b29a-40a2342d789c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: raw-sklearn-runtime-59568-predictor namespace: kserve-ci-e2e-test resourceVersion: "21008" uid: bc653b82-2e43-4173-af27-b173d21cddb2 kind: Event lastTimestamp: "2026-07-01T15:01:08Z" message: Scaled up replica set raw-sklearn-runtime-59568-predictor-6849b9cc8b from 0 to 1 metadata: creationTimestamp: "2026-07-01T15:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:08Z" name: raw-sklearn-runtime-59568-predictor.18be328972aaa9d5 namespace: kserve-ci-e2e-test resourceVersion: "21011" uid: 51288265-71da-4350-9371-93d4ff035cf6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:01:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-59568-predictor namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: 48472802-6919-47eb-b0bd-a49435cb6d3e kind: Event lastTimestamp: "2026-07-01T15:01:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:38Z" name: raw-sklearn-runtime-59568-predictor.18be328cf19486f2 namespace: kserve-ci-e2e-test resourceVersion: "21272" uid: d857b4f7-295c-4dc4-a5fc-009fc45e7b60 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:01:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-59568-predictor namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: 48472802-6919-47eb-b0bd-a49435cb6d3e kind: Event lastTimestamp: "2026-07-01T15:01:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-01T15:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:01:38Z" name: raw-sklearn-runtime-59568-predictor.18be328cf202f645 namespace: kserve-ci-e2e-test resourceVersion: "21273" uid: 747b271b-b7e5-450a-b679-5ca0ea5b77fe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T15:01:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-59568-predictor namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: 48472802-6919-47eb-b0bd-a49435cb6d3e kind: Event lastTimestamp: "2026-07-01T15:02:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:38Z" name: raw-sklearn-runtime-59568-predictor.18be3293f0841369 namespace: kserve-ci-e2e-test resourceVersion: "21703" uid: 03683967-787d-48ee-a2be-7edf79003260 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-01T15:01:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: raw-sklearn-runtime-59568-predictor namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: 48472802-6919-47eb-b0bd-a49435cb6d3e kind: Event lastTimestamp: "2026-07-01T15:02:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-01T15:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-01T15:02:38Z" name: raw-sklearn-runtime-59568-predictor.18be3293f0f693d0 namespace: kserve-ci-e2e-test resourceVersion: "21704" uid: a7d50e5d-bc78-41d1-a7c8-49ee76ff4fef reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-07-01T15:01:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-59568 namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: 3ed6ec7f-c231-4933-84cd-2419c9e37d7f kind: Event lastTimestamp: "2026-07-01T15:02:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-01T15:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:02:32Z" name: raw-sklearn-runtime-59568.18be328972aca673 namespace: kserve-ci-e2e-test resourceVersion: "21663" uid: 50c6cc91-9bd9-46dd-8153-59d18bcaff6c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-01T15:01:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-59568 namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: 3ed6ec7f-c231-4933-84cd-2419c9e37d7f kind: Event lastTimestamp: "2026-07-01T15:01:08Z" message: 'Failed to update status for InferenceService "raw-sklearn-runtime-59568": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-59568": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T15:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:01:08Z" name: raw-sklearn-runtime-59568.18be3289734a2d89 namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: 394cd0a4-2bb2-4192-adc6-a9ca0b6a4d99 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:01:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-59568 namespace: kserve-ci-e2e-test resourceVersion: "21003" uid: 3ed6ec7f-c231-4933-84cd-2419c9e37d7f kind: Event lastTimestamp: "2026-07-01T15:01:08Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "raw-sklearn-runtime-59568": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-01T15:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:01:08Z" name: raw-sklearn-runtime-59568.18be32897a72f7e5 namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: 5924657b-621b-4373-97a5-d97a1c361751 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-01T15:02:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: raw-sklearn-runtime-59568 namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 3ed6ec7f-c231-4933-84cd-2419c9e37d7f kind: Event lastTimestamp: "2026-07-01T15:02:32Z" message: InferenceService [raw-sklearn-runtime-59568] is Ready metadata: creationTimestamp: "2026-07-01T15:02:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-01T15:02:32Z" name: raw-sklearn-runtime-59568.18be329cf5d24861 namespace: kserve-ci-e2e-test resourceVersion: "21662" uid: 97e8d3cc-bb52-4777-887e-98274e41f46d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal kind: EventList metadata: resourceVersion: "21976"