--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:34:06.516204Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-155ee-7766bf4486-wp9ff namespace: kserve-ci-e2e-test resourceVersion: "20666" uid: b18c2fc2-81a9-4ce4-a454-d2d0a831c6ad kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-155ee-7766bf4486-wp9ff to ip-10-0-143-107.ec2.internal metadata: creationTimestamp: "2026-05-25T12:34:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:34:06Z" name: ensemble-graph-155ee-7766bf4486-wp9ff.18b2cf0a70f0a336 namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: 04af053a-bb1b-4468-879f-dcbfe8270192 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:34:06Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-155ee-7766bf4486-wp9ff namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: b18c2fc2-81a9-4ce4-a454-d2d0a831c6ad kind: Event lastTimestamp: "2026-05-25T12:34:06Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:34:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:34:06Z" name: ensemble-graph-155ee-7766bf4486-wp9ff.18b2cf0a89def98d namespace: kserve-ci-e2e-test resourceVersion: "20686" uid: a7b72c67-8717-4951-a258-8efd67a0d1ee reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:34:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-155ee} kind: Pod name: ensemble-graph-155ee-7766bf4486-wp9ff namespace: kserve-ci-e2e-test resourceVersion: "20667" uid: b18c2fc2-81a9-4ce4-a454-d2d0a831c6ad kind: Event lastTimestamp: "2026-05-25T12:34:06Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:41f0ca4b45114a1d81ffad66a667eba2baa4df82ad74c2ffc611dc9cdef0d94a" already present on machine metadata: creationTimestamp: "2026-05-25T12:34:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:34:06Z" name: ensemble-graph-155ee-7766bf4486-wp9ff.18b2cf0a8bc2e646 namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: 43664321-65c8-4e1b-81a4-d64c3e766a4c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:34:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-155ee} kind: Pod name: ensemble-graph-155ee-7766bf4486-wp9ff namespace: kserve-ci-e2e-test resourceVersion: "20667" uid: b18c2fc2-81a9-4ce4-a454-d2d0a831c6ad kind: Event lastTimestamp: "2026-05-25T12:34:07Z" message: 'Created container: ensemble-graph-155ee' metadata: creationTimestamp: "2026-05-25T12:34:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:34:07Z" name: ensemble-graph-155ee-7766bf4486-wp9ff.18b2cf0a8e6e5406 namespace: kserve-ci-e2e-test resourceVersion: "20689" uid: c7cbd8b8-e45d-4667-a5f0-bf3e9189968f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:34:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-155ee} kind: Pod name: ensemble-graph-155ee-7766bf4486-wp9ff namespace: kserve-ci-e2e-test resourceVersion: "20667" uid: b18c2fc2-81a9-4ce4-a454-d2d0a831c6ad kind: Event lastTimestamp: "2026-05-25T12:34:07Z" message: Started container ensemble-graph-155ee metadata: creationTimestamp: "2026-05-25T12:34:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:34:07Z" name: ensemble-graph-155ee-7766bf4486-wp9ff.18b2cf0a8edd8173 namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 6933eb1e-9d03-45a9-be77-1223fc7370b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-155ee} kind: Pod name: ensemble-graph-155ee-7766bf4486-wp9ff namespace: kserve-ci-e2e-test resourceVersion: "20667" uid: b18c2fc2-81a9-4ce4-a454-d2d0a831c6ad kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: Stopping container ensemble-graph-155ee metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:21Z" name: ensemble-graph-155ee-7766bf4486-wp9ff.18b2cf7d97091738 namespace: kserve-ci-e2e-test resourceVersion: "24167" uid: e1b6a1db-5e6c-4391-8d92-0b69e4b5eea1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:42:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-155ee} kind: Pod name: ensemble-graph-155ee-7766bf4486-wp9ff namespace: kserve-ci-e2e-test resourceVersion: "20667" uid: b18c2fc2-81a9-4ce4-a454-d2d0a831c6ad kind: Event lastTimestamp: "2026-05-25T12:42:48Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T12:42:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:48Z" name: ensemble-graph-155ee-7766bf4486-wp9ff.18b2cf7e2b28cc9f namespace: kserve-ci-e2e-test resourceVersion: "24502" uid: 44f90bad-6842-44b6-a9d9-de1ebea39234 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:34:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-155ee-7766bf4486 namespace: kserve-ci-e2e-test resourceVersion: "20663" uid: a1fd0533-0e45-4136-b85e-a9f2054bee8c kind: Event lastTimestamp: "2026-05-25T12:34:06Z" message: 'Created pod: ensemble-graph-155ee-7766bf4486-wp9ff' metadata: creationTimestamp: "2026-05-25T12:34:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:34:06Z" name: ensemble-graph-155ee-7766bf4486.18b2cf0a707c187a namespace: kserve-ci-e2e-test resourceVersion: "20669" uid: cd8b3e79-3cab-4079-9f67-bb2cbe16ea40 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:34:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-155ee namespace: kserve-ci-e2e-test resourceVersion: "20661" uid: 598d7f8e-fe1c-495f-a5ff-40892c3f1893 kind: Event lastTimestamp: "2026-05-25T12:34:06Z" message: Scaled up replica set ensemble-graph-155ee-7766bf4486 from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:34:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:34:06Z" name: ensemble-graph-155ee.18b2cf0a6e8feae2 namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: 705b5897-3097-470a-8872-59874a8cb98b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:34:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-155ee namespace: kserve-ci-e2e-test resourceVersion: "20681" uid: 3c14d3c3-e98d-459f-832a-f6efb47b75b2 kind: Event lastTimestamp: "2026-05-25T12:34:06Z" message: InferenceGraph [ensemble-graph-155ee] is Ready metadata: creationTimestamp: "2026-05-25T12:34:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:34:06Z" name: ensemble-graph-155ee.18b2cf0a77d4ea3d namespace: kserve-ci-e2e-test resourceVersion: "20683" uid: 2caf039e-3130-492e-a5ac-d3305ff49ec3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:34:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-155ee namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: e4b2ac32-b6ee-4211-ba8e-16c6ea8554bf kind: Event lastTimestamp: "2026-05-25T12:34:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:34:51Z" name: ensemble-graph-155ee.18b2cf0ded0c32c1 namespace: kserve-ci-e2e-test resourceVersion: "21088" uid: 30d57890-3ba2-4044-9ec8-b31cb666873c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:34:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-155ee namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: e4b2ac32-b6ee-4211-ba8e-16c6ea8554bf kind: Event lastTimestamp: "2026-05-25T12:34:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:34:51Z" name: ensemble-graph-155ee.18b2cf0ded9fabcc namespace: kserve-ci-e2e-test resourceVersion: "21089" uid: 2e121f73-1fc9-406f-a217-950cc2c98a20 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:32:46.070591Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-bcff4-76cb6fc4b6-xrpnv namespace: kserve-ci-e2e-test resourceVersion: "19608" uid: 2bb89a39-f92b-4976-8048-d2a5d7971dd2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-bcff4-76cb6fc4b6-xrpnv to ip-10-0-143-107.ec2.internal metadata: creationTimestamp: "2026-05-25T12:32:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:32:46Z" name: ensemble-graph-bcff4-76cb6fc4b6-xrpnv.18b2cef7b60208b5 namespace: kserve-ci-e2e-test resourceVersion: "19615" uid: 7891379f-fa36-4213-bbdf-65e1df3a3479 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:46Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-bcff4-76cb6fc4b6-xrpnv namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 2bb89a39-f92b-4976-8048-d2a5d7971dd2 kind: Event lastTimestamp: "2026-05-25T12:32:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-bcff4-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:32:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:46Z" name: ensemble-graph-bcff4-76cb6fc4b6-xrpnv.18b2cef7c661b897 namespace: kserve-ci-e2e-test resourceVersion: "19633" uid: 99890c4f-97ed-4ae3-a7a4-06a1cde0671b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:47Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-bcff4-76cb6fc4b6-xrpnv namespace: kserve-ci-e2e-test resourceVersion: "19618" uid: 2bb89a39-f92b-4976-8048-d2a5d7971dd2 kind: Event lastTimestamp: "2026-05-25T12:32:47Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:32:47Z" name: ensemble-graph-bcff4-76cb6fc4b6-xrpnv.18b2cef7f2bab37b namespace: kserve-ci-e2e-test resourceVersion: "19640" uid: 35ced632-2b28-4d7a-be1d-9a4cbf717a8b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-bcff4} kind: Pod name: ensemble-graph-bcff4-76cb6fc4b6-xrpnv namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 2bb89a39-f92b-4976-8048-d2a5d7971dd2 kind: Event lastTimestamp: "2026-05-25T12:32:47Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:41f0ca4b45114a1d81ffad66a667eba2baa4df82ad74c2ffc611dc9cdef0d94a" already present on machine metadata: creationTimestamp: "2026-05-25T12:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:47Z" name: ensemble-graph-bcff4-76cb6fc4b6-xrpnv.18b2cef7f4af7a3d namespace: kserve-ci-e2e-test resourceVersion: "19643" uid: df11a1b6-074b-454c-bfc1-821c7d1ac493 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-bcff4} kind: Pod name: ensemble-graph-bcff4-76cb6fc4b6-xrpnv namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 2bb89a39-f92b-4976-8048-d2a5d7971dd2 kind: Event lastTimestamp: "2026-05-25T12:32:47Z" message: 'Created container: ensemble-graph-bcff4' metadata: creationTimestamp: "2026-05-25T12:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:47Z" name: ensemble-graph-bcff4-76cb6fc4b6-xrpnv.18b2cef7f7788875 namespace: kserve-ci-e2e-test resourceVersion: "19645" uid: 59eaa19c-8afe-4d46-a3d2-620298c66065 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-bcff4} kind: Pod name: ensemble-graph-bcff4-76cb6fc4b6-xrpnv namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 2bb89a39-f92b-4976-8048-d2a5d7971dd2 kind: Event lastTimestamp: "2026-05-25T12:32:47Z" message: Started container ensemble-graph-bcff4 metadata: creationTimestamp: "2026-05-25T12:32:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:47Z" name: ensemble-graph-bcff4-76cb6fc4b6-xrpnv.18b2cef7f7ea3667 namespace: kserve-ci-e2e-test resourceVersion: "19647" uid: f7a6171e-6195-4461-b14c-2a29b5006cb6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-bcff4} kind: Pod name: ensemble-graph-bcff4-76cb6fc4b6-xrpnv namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 2bb89a39-f92b-4976-8048-d2a5d7971dd2 kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: Stopping container ensemble-graph-bcff4 metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:56Z" name: ensemble-graph-bcff4-76cb6fc4b6-xrpnv.18b2cefa0f2c7e4a namespace: kserve-ci-e2e-test resourceVersion: "19735" uid: beebee7d-2113-46d2-8065-cd75bdd38bd2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:32:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-bcff4} kind: Pod name: ensemble-graph-bcff4-76cb6fc4b6-xrpnv namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 2bb89a39-f92b-4976-8048-d2a5d7971dd2 kind: Event lastTimestamp: "2026-05-25T12:33:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T12:32:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:23Z" name: ensemble-graph-bcff4-76cb6fc4b6-xrpnv.18b2cefa905f3d6d namespace: kserve-ci-e2e-test resourceVersion: "20095" uid: 659a490e-b03f-4e4c-84f4-ccfe3581f713 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-bcff4-76cb6fc4b6 namespace: kserve-ci-e2e-test resourceVersion: "19602" uid: 921b3b91-f3a2-4b89-b600-3cd7beaf4369 kind: Event lastTimestamp: "2026-05-25T12:32:46Z" message: 'Created pod: ensemble-graph-bcff4-76cb6fc4b6-xrpnv' metadata: creationTimestamp: "2026-05-25T12:32:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:32:46Z" name: ensemble-graph-bcff4-76cb6fc4b6.18b2cef7b568546a namespace: kserve-ci-e2e-test resourceVersion: "19611" uid: b97a13fc-14de-465b-96fd-dab6a0bfbacc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-bcff4 namespace: kserve-ci-e2e-test resourceVersion: "19597" uid: 50cbc6cf-74ec-4ed5-a587-d25c1b06c938 kind: Event lastTimestamp: "2026-05-25T12:32:46Z" message: Scaled up replica set ensemble-graph-bcff4-76cb6fc4b6 from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:32:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:32:46Z" name: ensemble-graph-bcff4.18b2cef7b4187ed0 namespace: kserve-ci-e2e-test resourceVersion: "19605" uid: 01b9699a-bc79-4cd9-b9c3-1c2d8d340e1d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-bcff4 namespace: kserve-ci-e2e-test resourceVersion: "19620" uid: 70790d15-7913-47c2-9b22-58f07a671d61 kind: Event lastTimestamp: "2026-05-25T12:32:46Z" message: InferenceGraph [ensemble-graph-bcff4] is Ready metadata: creationTimestamp: "2026-05-25T12:32:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:32:46Z" name: ensemble-graph-bcff4.18b2cef7b969eddc namespace: kserve-ci-e2e-test resourceVersion: "19623" uid: 53e18903-6c8c-496b-9980-ebb1e6d9fbf8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:42:51.323032Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq namespace: kserve-ci-e2e-test resourceVersion: "24614" uid: 017fafb4-c256-47d2-b271-a6c2feff2636 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq to ip-10-0-142-53.ec2.internal metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:42:51Z" name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq.18b2cf84a1dd60db namespace: kserve-ci-e2e-test resourceVersion: "24620" uid: aa2a7657-b8ea-44a2-82b0-ba85cc6456c5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq namespace: kserve-ci-e2e-test resourceVersion: "24625" uid: 017fafb4-c256-47d2-b271-a6c2feff2636 kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: Add eth0 [10.134.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:42:51Z" name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq.18b2cf84bc58433b namespace: kserve-ci-e2e-test resourceVersion: "24639" uid: edfeffca-d717-46d7-88d5-a0ed8709cbb2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq namespace: kserve-ci-e2e-test resourceVersion: "24616" uid: 017fafb4-c256-47d2-b271-a6c2feff2636 kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:51Z" name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq.18b2cf84bdeef29c namespace: kserve-ci-e2e-test resourceVersion: "24641" uid: e02d351f-06a6-47f9-ab9e-a5b2825abd47 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq namespace: kserve-ci-e2e-test resourceVersion: "24616" uid: 017fafb4-c256-47d2-b271-a6c2feff2636 kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:51Z" name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq.18b2cf84c0eced59 namespace: kserve-ci-e2e-test resourceVersion: "24642" uid: 4b6f5451-8885-4585-978d-daf15114e2fc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq namespace: kserve-ci-e2e-test resourceVersion: "24616" uid: 017fafb4-c256-47d2-b271-a6c2feff2636 kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:51Z" name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq.18b2cf84c1613d1d namespace: kserve-ci-e2e-test resourceVersion: "24643" uid: 54f6ab41-fe91-4834-b948-6e221609ce33 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq namespace: kserve-ci-e2e-test resourceVersion: "24616" uid: 017fafb4-c256-47d2-b271-a6c2feff2636 kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:51Z" name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq.18b2cf84c16f4e0b namespace: kserve-ci-e2e-test resourceVersion: "24644" uid: be0bd6a4-3ad1-4982-954d-15b38dcdac67 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq namespace: kserve-ci-e2e-test resourceVersion: "24616" uid: 017fafb4-c256-47d2-b271-a6c2feff2636 kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:51Z" name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq.18b2cf84c5132b98 namespace: kserve-ci-e2e-test resourceVersion: "24645" uid: 1d8fa731-d7ae-4ff0-b2a3-78922065ff39 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq namespace: kserve-ci-e2e-test resourceVersion: "24616" uid: 017fafb4-c256-47d2-b271-a6c2feff2636 kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:51Z" name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq.18b2cf84c5922307 namespace: kserve-ci-e2e-test resourceVersion: "24646" uid: 94bcb93b-bc56-4c95-8e8c-6f0858f75e55 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:42:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq namespace: kserve-ci-e2e-test resourceVersion: "24616" uid: 017fafb4-c256-47d2-b271-a6c2feff2636 kind: Event lastTimestamp: "2026-05-25T12:43:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.51:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:42:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:29Z" name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq.18b2cf8524673698 namespace: kserve-ci-e2e-test resourceVersion: "25030" uid: 701529ef-bae5-47d1-b959-26fc797db0ad reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:00:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq namespace: kserve-ci-e2e-test resourceVersion: "24616" uid: 017fafb4-c256-47d2-b271-a6c2feff2636 kind: Event lastTimestamp: "2026-05-25T13:00:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:00:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:00:20Z" name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq.18b2d078fb37ad5f namespace: kserve-ci-e2e-test resourceVersion: "32662" uid: a5ccaa72-f8ae-4028-b7b6-a443b50aea54 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:00:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq namespace: kserve-ci-e2e-test resourceVersion: "24616" uid: 017fafb4-c256-47d2-b271-a6c2feff2636 kind: Event lastTimestamp: "2026-05-25T13:00:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:00:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:00:20Z" name: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq.18b2d078fb3818d5 namespace: kserve-ci-e2e-test resourceVersion: "32663" uid: 56509140-3066-494d-a905-2e1f07d3e139 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-09598-predictor-7c4d94fb5f namespace: kserve-ci-e2e-test resourceVersion: "24607" uid: 8e81d9ba-0000-4a27-b014-bb6024950695 kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: 'Created pod: error-404-isvc-09598-predictor-7c4d94fb5f-hqnbq' metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:42:51Z" name: error-404-isvc-09598-predictor-7c4d94fb5f.18b2cf84a155d8fa namespace: kserve-ci-e2e-test resourceVersion: "24615" uid: 5a2ec056-48ad-4573-a350-68d84d11f431 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-09598-predictor namespace: kserve-ci-e2e-test resourceVersion: "24603" uid: df588d1e-56c2-4491-b7c9-79e173fbab9d kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: Scaled up replica set error-404-isvc-09598-predictor-7c4d94fb5f from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:42:51Z" name: error-404-isvc-09598-predictor.18b2cf849e7552af namespace: kserve-ci-e2e-test resourceVersion: "24609" uid: 053fbb0a-39d2-41d6-a745-bf0250c181de reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:43:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-09598-predictor namespace: kserve-ci-e2e-test resourceVersion: "24605" uid: bf8ddcff-1732-4d50-a38f-5959f55947ae kind: Event lastTimestamp: "2026-05-25T12:43:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:43:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:43:21Z" name: error-404-isvc-09598-predictor.18b2cf881d0c04bb namespace: kserve-ci-e2e-test resourceVersion: "24931" uid: c57d36ac-9a21-4602-8054-3c325200416b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:43:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-09598-predictor namespace: kserve-ci-e2e-test resourceVersion: "24605" uid: bf8ddcff-1732-4d50-a38f-5959f55947ae kind: Event lastTimestamp: "2026-05-25T12:43:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:43:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:43:21Z" name: error-404-isvc-09598-predictor.18b2cf881d7b0c2c namespace: kserve-ci-e2e-test resourceVersion: "24934" uid: 2ef772ea-51c0-4af6-8e46-1169110a8598 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:43:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-09598-predictor namespace: kserve-ci-e2e-test resourceVersion: "24790" uid: bf8ddcff-1732-4d50-a38f-5959f55947ae kind: Event lastTimestamp: "2026-05-25T12:44:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:44:06Z" name: error-404-isvc-09598-predictor.18b2cf8f1afe3a70 namespace: kserve-ci-e2e-test resourceVersion: "25531" uid: 9f8943cb-43b6-4fcb-96d5-92b9a2ebd25f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:43:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-09598-predictor namespace: kserve-ci-e2e-test resourceVersion: "24790" uid: bf8ddcff-1732-4d50-a38f-5959f55947ae kind: Event lastTimestamp: "2026-05-25T12:44:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:44:06Z" name: error-404-isvc-09598-predictor.18b2cf8f1b731be3 namespace: kserve-ci-e2e-test resourceVersion: "25533" uid: 1311bcb4-f9cd-4450-89ca-2aa3b8c53985 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-09598 namespace: kserve-ci-e2e-test resourceVersion: "24572" uid: e7154039-8afa-4677-b50d-ef9281c56c4d kind: Event lastTimestamp: "2026-05-25T12:43:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:43:39Z" name: error-404-isvc-09598.18b2cf849e2c40a4 namespace: kserve-ci-e2e-test resourceVersion: "25269" uid: b3525e1b-02ed-40c4-a647-9a0d57746c27 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-09598 namespace: kserve-ci-e2e-test resourceVersion: "24572" uid: e7154039-8afa-4677-b50d-ef9281c56c4d kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: 'Failed to update status for InferenceService "error-404-isvc-09598": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-09598": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:42:51Z" name: error-404-isvc-09598.18b2cf849ead79cd namespace: kserve-ci-e2e-test resourceVersion: "24611" uid: 109e52ed-7e61-4498-9de8-1a343eeaa623 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-09598 namespace: kserve-ci-e2e-test resourceVersion: "24572" uid: e7154039-8afa-4677-b50d-ef9281c56c4d kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-09598": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:42:51Z" name: error-404-isvc-09598.18b2cf849eade6c2 namespace: kserve-ci-e2e-test resourceVersion: "24612" uid: b4846cfa-3bbc-4fc8-9c79-b5e3f5ecdeb5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-09598 namespace: kserve-ci-e2e-test resourceVersion: "25267" uid: e7154039-8afa-4677-b50d-ef9281c56c4d kind: Event lastTimestamp: "2026-05-25T12:43:39Z" message: InferenceService [error-404-isvc-09598] is Ready metadata: creationTimestamp: "2026-05-25T12:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:43:39Z" name: error-404-isvc-09598.18b2cf8fe331215a namespace: kserve-ci-e2e-test resourceVersion: "25268" uid: 53aac0c3-c957-4865-bf50-f111b02b1ef6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:32:56.532462Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n namespace: kserve-ci-e2e-test resourceVersion: "19813" uid: 3f43e011-dd0d-4717-8d72-3e2882bb8650 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-155ee-predictor-57c99b6f99-mtr6n to ip-10-0-142-53.ec2.internal metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:32:56Z" name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n.18b2cefa2595885a namespace: kserve-ci-e2e-test resourceVersion: "19817" uid: 2e416d4b-9fb9-42ec-9d8c-0c5f0ea65e27 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n namespace: kserve-ci-e2e-test resourceVersion: "19823" uid: 3f43e011-dd0d-4717-8d72-3e2882bb8650 kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:32:56Z" name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n.18b2cefa40aede97 namespace: kserve-ci-e2e-test resourceVersion: "19838" uid: b8b1671a-eba2-4983-93f0-58e16e5d87e4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n namespace: kserve-ci-e2e-test resourceVersion: "19814" uid: 3f43e011-dd0d-4717-8d72-3e2882bb8650 kind: Event lastTimestamp: "2026-05-25T12:32:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T12:32:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:57Z" name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n.18b2cefa42b67c53 namespace: kserve-ci-e2e-test resourceVersion: "19842" uid: 21ed375d-aab8-4646-b330-186a6b3e4807 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n namespace: kserve-ci-e2e-test resourceVersion: "19814" uid: 3f43e011-dd0d-4717-8d72-3e2882bb8650 kind: Event lastTimestamp: "2026-05-25T12:32:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:32:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:57Z" name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n.18b2cefa45d5b22c namespace: kserve-ci-e2e-test resourceVersion: "19843" uid: d50a5be5-a942-4960-ab0a-86a8bb0f7c73 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n namespace: kserve-ci-e2e-test resourceVersion: "19814" uid: 3f43e011-dd0d-4717-8d72-3e2882bb8650 kind: Event lastTimestamp: "2026-05-25T12:32:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:32:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:57Z" name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n.18b2cefa464c3e70 namespace: kserve-ci-e2e-test resourceVersion: "19844" uid: 5b7324a0-7164-4bda-96a6-664e73eabde0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n namespace: kserve-ci-e2e-test resourceVersion: "19814" uid: 3f43e011-dd0d-4717-8d72-3e2882bb8650 kind: Event lastTimestamp: "2026-05-25T12:32:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T12:32:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:57Z" name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n.18b2cefa465853ff namespace: kserve-ci-e2e-test resourceVersion: "19845" uid: 73561fa1-0509-41fa-9768-c6f2e3bae1fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n namespace: kserve-ci-e2e-test resourceVersion: "19814" uid: 3f43e011-dd0d-4717-8d72-3e2882bb8650 kind: Event lastTimestamp: "2026-05-25T12:32:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:32:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:57Z" name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n.18b2cefa4a246188 namespace: kserve-ci-e2e-test resourceVersion: "19848" uid: 750a0c54-0bcb-4ff0-a4e3-9eb5102ece81 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n namespace: kserve-ci-e2e-test resourceVersion: "19814" uid: 3f43e011-dd0d-4717-8d72-3e2882bb8650 kind: Event lastTimestamp: "2026-05-25T12:32:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:32:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:57Z" name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n.18b2cefa4a9e8430 namespace: kserve-ci-e2e-test resourceVersion: "19850" uid: 1979cc4e-7a26-4201-be91-ff2c1e0018d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-25T12:32:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n namespace: kserve-ci-e2e-test resourceVersion: "19814" uid: 3f43e011-dd0d-4717-8d72-3e2882bb8650 kind: Event lastTimestamp: "2026-05-25T12:42:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:32:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:24Z" name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n.18b2cefa8bd44efd namespace: kserve-ci-e2e-test resourceVersion: "24312" uid: 5d1c370e-1b3a-4441-b77c-2abad61c9d59 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n namespace: kserve-ci-e2e-test resourceVersion: "19814" uid: 3f43e011-dd0d-4717-8d72-3e2882bb8650 kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:21Z" name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n.18b2cf7da2139491 namespace: kserve-ci-e2e-test resourceVersion: "24221" uid: ea581221-2199-4c56-baee-d438086aed93 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n namespace: kserve-ci-e2e-test resourceVersion: "19814" uid: 3f43e011-dd0d-4717-8d72-3e2882bb8650 kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:21Z" name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n.18b2cf7da21439f1 namespace: kserve-ci-e2e-test resourceVersion: "24224" uid: f137260b-b9ad-4ede-a344-9bde068d02b0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n namespace: kserve-ci-e2e-test resourceVersion: "19814" uid: 3f43e011-dd0d-4717-8d72-3e2882bb8650 kind: Event lastTimestamp: "2026-05-25T12:42:24Z" message: 'Readiness probe failed: Get "https://10.134.0.45:8643/healthz": dial tcp 10.134.0.45:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:24Z" name: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n.18b2cf7e5468a405 namespace: kserve-ci-e2e-test resourceVersion: "24309" uid: 5702da5c-4ad3-46a0-9cce-82b7992499d2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-155ee-predictor-57c99b6f99 namespace: kserve-ci-e2e-test resourceVersion: "19804" uid: 357dc196-8dc4-419e-9ebd-3963455659de kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: 'Created pod: error-404-isvc-155ee-predictor-57c99b6f99-mtr6n' metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:32:56Z" name: error-404-isvc-155ee-predictor-57c99b6f99.18b2cefa2512b692 namespace: kserve-ci-e2e-test resourceVersion: "19815" uid: ece05b45-0d6f-4968-b03d-087795a958eb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-155ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "19801" uid: f109f642-9171-4de0-844c-72b90013b27c kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: Scaled up replica set error-404-isvc-155ee-predictor-57c99b6f99 from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:32:56Z" name: error-404-isvc-155ee-predictor.18b2cefa22de65ad namespace: kserve-ci-e2e-test resourceVersion: "19807" uid: f03cd7ab-b23f-4abc-be3c-a5a4d2249818 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-155ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: aebd618f-7310-4b40-903e-edcbaba16e86 kind: Event lastTimestamp: "2026-05-25T12:33:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:33:11Z" name: error-404-isvc-155ee-predictor.18b2cefda15e8b20 namespace: kserve-ci-e2e-test resourceVersion: "19994" uid: eb99648a-b5fb-44aa-aa24-9a7a71572f1f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-155ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: aebd618f-7310-4b40-903e-edcbaba16e86 kind: Event lastTimestamp: "2026-05-25T12:33:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:33:11Z" name: error-404-isvc-155ee-predictor.18b2cefda1e07cf5 namespace: kserve-ci-e2e-test resourceVersion: "19996" uid: 55c44583-a784-4b1f-8913-0a27139cf70b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:33:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-155ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "19995" uid: aebd618f-7310-4b40-903e-edcbaba16e86 kind: Event lastTimestamp: "2026-05-25T12:34:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:33:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:34:26Z" name: error-404-isvc-155ee-predictor.18b2cf0120ad3330 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 297024a4-eac5-4429-9fa9-9c26a6e3b25d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:33:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-155ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "19995" uid: aebd618f-7310-4b40-903e-edcbaba16e86 kind: Event lastTimestamp: "2026-05-25T12:34:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:33:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:34:26Z" name: error-404-isvc-155ee-predictor.18b2cf01212779ec namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: bf8461a9-5d7e-491c-b597-1b78f8855c9a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-155ee namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 5515292a-7f43-4f5a-a80d-560958345c53 kind: Event lastTimestamp: "2026-05-25T12:33:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:33:54Z" name: error-404-isvc-155ee.18b2cefa22ac4142 namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: f9aa8ca7-82af-4b7b-8778-9136db0a06da reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-155ee namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 5515292a-7f43-4f5a-a80d-560958345c53 kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: 'Failed to update status for InferenceService "error-404-isvc-155ee": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-155ee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:32:56Z" name: error-404-isvc-155ee.18b2cefa23446eca namespace: kserve-ci-e2e-test resourceVersion: "19809" uid: 676e6c3b-e783-44d3-b471-6b2101d260bd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-155ee namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 5515292a-7f43-4f5a-a80d-560958345c53 kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-155ee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:32:56Z" name: error-404-isvc-155ee.18b2cefa2344fab7 namespace: kserve-ci-e2e-test resourceVersion: "19811" uid: c69b4a2b-88b1-4137-904e-551e3c5cc370 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-155ee namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: 5515292a-7f43-4f5a-a80d-560958345c53 kind: Event lastTimestamp: "2026-05-25T12:33:54Z" message: InferenceService [error-404-isvc-155ee] is Ready metadata: creationTimestamp: "2026-05-25T12:33:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:33:54Z" name: error-404-isvc-155ee.18b2cf07a018417f namespace: kserve-ci-e2e-test resourceVersion: "20548" uid: 790b59fe-6a4e-4c19-9252-c58291c8a5e9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:21:01.104733Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56 namespace: kserve-ci-e2e-test resourceVersion: "13438" uid: dbeb1945-c642-49c8-ab81-19aeffc568d3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1742e-predictor-5764c6c5cf-6wk56 to ip-10-0-142-53.ec2.internal metadata: creationTimestamp: "2026-05-25T12:21:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:21:01Z" name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56.18b2ce5392c4afc3 namespace: kserve-ci-e2e-test resourceVersion: "13443" uid: 47330e2b-b271-4240-a212-2a46be6c08a5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:21:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56 namespace: kserve-ci-e2e-test resourceVersion: "13440" uid: dbeb1945-c642-49c8-ab81-19aeffc568d3 kind: Event lastTimestamp: "2026-05-25T12:21:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-1742e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:21:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:01Z" name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56.18b2ce539a63cb27 namespace: kserve-ci-e2e-test resourceVersion: "13495" uid: 726f7695-b0bf-4f08-aeb9-dc595edd791c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56 namespace: kserve-ci-e2e-test resourceVersion: "13447" uid: dbeb1945-c642-49c8-ab81-19aeffc568d3 kind: Event lastTimestamp: "2026-05-25T12:21:03Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:21:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:21:03Z" name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56.18b2ce5406e06484 namespace: kserve-ci-e2e-test resourceVersion: "13542" uid: 84f928c2-d89d-4680-841d-45b1c1a34074 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56 namespace: kserve-ci-e2e-test resourceVersion: "13440" uid: dbeb1945-c642-49c8-ab81-19aeffc568d3 kind: Event lastTimestamp: "2026-05-25T12:21:03Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" metadata: creationTimestamp: "2026-05-25T12:21:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:03Z" name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56.18b2ce540932c5c1 namespace: kserve-ci-e2e-test resourceVersion: "13544" uid: c787c368-a8e5-43b9-a17d-323fd2ecad1e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56 namespace: kserve-ci-e2e-test resourceVersion: "13440" uid: dbeb1945-c642-49c8-ab81-19aeffc568d3 kind: Event lastTimestamp: "2026-05-25T12:21:17Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" in 14.23s (14.23s including waiting). Image size: 1334724715 bytes.' metadata: creationTimestamp: "2026-05-25T12:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:17Z" name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56.18b2ce5759613e91 namespace: kserve-ci-e2e-test resourceVersion: "13708" uid: a3c0b8e8-8ee6-447a-9203-f836c244f6d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56 namespace: kserve-ci-e2e-test resourceVersion: "13440" uid: dbeb1945-c642-49c8-ab81-19aeffc568d3 kind: Event lastTimestamp: "2026-05-25T12:21:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:17Z" name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56.18b2ce575f675236 namespace: kserve-ci-e2e-test resourceVersion: "13714" uid: 45edba13-69ad-44c3-baf4-53b2736b8c9c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56 namespace: kserve-ci-e2e-test resourceVersion: "13440" uid: dbeb1945-c642-49c8-ab81-19aeffc568d3 kind: Event lastTimestamp: "2026-05-25T12:21:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:17Z" name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56.18b2ce575fda57a0 namespace: kserve-ci-e2e-test resourceVersion: "13717" uid: 3ceec23f-ed69-47c1-b0df-d69efece742f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56 namespace: kserve-ci-e2e-test resourceVersion: "13440" uid: dbeb1945-c642-49c8-ab81-19aeffc568d3 kind: Event lastTimestamp: "2026-05-25T12:21:17Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-25T12:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:17Z" name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56.18b2ce575fe73149 namespace: kserve-ci-e2e-test resourceVersion: "13719" uid: c9f7d7ff-174e-4291-8d30-c778a8589851 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56 namespace: kserve-ci-e2e-test resourceVersion: "13440" uid: dbeb1945-c642-49c8-ab81-19aeffc568d3 kind: Event lastTimestamp: "2026-05-25T12:21:19Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.548s (2.548s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-25T12:21:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:19Z" name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56.18b2ce57f7d3201b namespace: kserve-ci-e2e-test resourceVersion: "13738" uid: 0d199e62-870f-4003-860d-a3c6c4a4a4d5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56 namespace: kserve-ci-e2e-test resourceVersion: "13440" uid: dbeb1945-c642-49c8-ab81-19aeffc568d3 kind: Event lastTimestamp: "2026-05-25T12:21:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:20Z" name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56.18b2ce57fbf70d55 namespace: kserve-ci-e2e-test resourceVersion: "13741" uid: 55995c8f-d55d-4a92-bfbb-bba93ea04671 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56 namespace: kserve-ci-e2e-test resourceVersion: "13440" uid: dbeb1945-c642-49c8-ab81-19aeffc568d3 kind: Event lastTimestamp: "2026-05-25T12:21:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:20Z" name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56.18b2ce57fc6e322f namespace: kserve-ci-e2e-test resourceVersion: "13743" uid: a413da38-075e-4e88-b97e-b03a7bb27914 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:21:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56 namespace: kserve-ci-e2e-test resourceVersion: "13440" uid: dbeb1945-c642-49c8-ab81-19aeffc568d3 kind: Event lastTimestamp: "2026-05-25T12:21:56Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:56Z" name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56.18b2ce5821cec308 namespace: kserve-ci-e2e-test resourceVersion: "14078" uid: aee3c54e-01c9-4324-96ee-d4d55cc60b7e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56 namespace: kserve-ci-e2e-test resourceVersion: "13440" uid: dbeb1945-c642-49c8-ab81-19aeffc568d3 kind: Event lastTimestamp: "2026-05-25T12:22:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:22:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:30Z" name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56.18b2ce687cc3133d namespace: kserve-ci-e2e-test resourceVersion: "14460" uid: cc563b9e-7506-435d-b67f-53c8d6eab0ce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56 namespace: kserve-ci-e2e-test resourceVersion: "13440" uid: dbeb1945-c642-49c8-ab81-19aeffc568d3 kind: Event lastTimestamp: "2026-05-25T12:22:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:22:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:30Z" name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56.18b2ce687cc3df40 namespace: kserve-ci-e2e-test resourceVersion: "14461" uid: 8cfe3ca9-2bb7-4774-9a0c-36fa7ffe8e7b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56 namespace: kserve-ci-e2e-test resourceVersion: "13440" uid: dbeb1945-c642-49c8-ab81-19aeffc568d3 kind: Event lastTimestamp: "2026-05-25T12:22:31Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:31Z" name: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56.18b2ce68a9f5e2a0 namespace: kserve-ci-e2e-test resourceVersion: "14546" uid: a8706671-ba15-4734-842a-c41090c9d0a6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1742e-predictor-5764c6c5cf namespace: kserve-ci-e2e-test resourceVersion: "13426" uid: 853c64ed-41ae-496b-bd72-fa0ab0cdddba kind: Event lastTimestamp: "2026-05-25T12:21:01Z" message: 'Created pod: error-404-isvc-1742e-predictor-5764c6c5cf-6wk56' metadata: creationTimestamp: "2026-05-25T12:21:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:21:01Z" name: error-404-isvc-1742e-predictor-5764c6c5cf.18b2ce539248897c namespace: kserve-ci-e2e-test resourceVersion: "13439" uid: 065c1320-6f93-423e-8dc4-5c8269abefbc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1742e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13421" uid: 535f5565-5038-4fc7-a6ce-bc2d47c6e09c kind: Event lastTimestamp: "2026-05-25T12:21:01Z" message: Scaled up replica set error-404-isvc-1742e-predictor-5764c6c5cf from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:21:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:21:01Z" name: error-404-isvc-1742e-predictor.18b2ce53906eab51 namespace: kserve-ci-e2e-test resourceVersion: "13428" uid: 9b7c9bcb-edd6-4a5b-9c63-8ddc249dcee1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:21:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1742e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13425" uid: ea33ef22-81b8-4684-aac3-86aac0602dc5 kind: Event lastTimestamp: "2026-05-25T12:22:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:21:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:01Z" name: error-404-isvc-1742e-predictor.18b2ce570f9f2c45 namespace: kserve-ci-e2e-test resourceVersion: "14116" uid: 080775ee-a14f-4b15-8669-2766e53b07af reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:21:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1742e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13425" uid: ea33ef22-81b8-4684-aac3-86aac0602dc5 kind: Event lastTimestamp: "2026-05-25T12:22:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:21:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:01Z" name: error-404-isvc-1742e-predictor.18b2ce57103d17b9 namespace: kserve-ci-e2e-test resourceVersion: "14117" uid: d7b20631-4fc7-4bf1-92df-28f0a3cdcc86 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:21:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1742e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13689" uid: ea33ef22-81b8-4684-aac3-86aac0602dc5 kind: Event lastTimestamp: "2026-05-25T12:22:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:16Z" name: error-404-isvc-1742e-predictor.18b2ce5e0db398e3 namespace: kserve-ci-e2e-test resourceVersion: "14249" uid: 0d7261bc-e71d-41e7-951f-a4465922e3be reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:21:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1742e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13689" uid: ea33ef22-81b8-4684-aac3-86aac0602dc5 kind: Event lastTimestamp: "2026-05-25T12:22:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:21:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:16Z" name: error-404-isvc-1742e-predictor.18b2ce5e0e24d2c4 namespace: kserve-ci-e2e-test resourceVersion: "14250" uid: 89ac29a7-f6da-4ddc-80f7-0ebdfc9f40b9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:21:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1742e namespace: kserve-ci-e2e-test resourceVersion: "13412" uid: 07506894-807a-4d3d-84c0-3246c7d8ff94 kind: Event lastTimestamp: "2026-05-25T12:22:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:21:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:22:06Z" name: error-404-isvc-1742e.18b2ce5390774b54 namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: 4259bca9-ffa0-4f5a-b159-20a0f92281c8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1742e namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: 07506894-807a-4d3d-84c0-3246c7d8ff94 kind: Event lastTimestamp: "2026-05-25T12:22:06Z" message: InferenceService [error-404-isvc-1742e] is Ready metadata: creationTimestamp: "2026-05-25T12:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:22:06Z" name: error-404-isvc-1742e.18b2ce62de701c1b namespace: kserve-ci-e2e-test resourceVersion: "14170" uid: d96eefba-98b4-4a42-8b37-61dc4969fa36 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:22:31.120533Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd namespace: kserve-ci-e2e-test resourceVersion: "14520" uid: 771a87fe-641c-40bd-8588-a6241156f7b0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-40bda-predictor-6664b4979f-g8jxd to ip-10-0-142-53.ec2.internal metadata: creationTimestamp: "2026-05-25T12:22:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:22:31Z" name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd.18b2ce688820cabf namespace: kserve-ci-e2e-test resourceVersion: "14524" uid: 7dd0ff9e-d401-43a9-9151-d25b5e4b7c60 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd namespace: kserve-ci-e2e-test resourceVersion: "14522" uid: 771a87fe-641c-40bd-8588-a6241156f7b0 kind: Event lastTimestamp: "2026-05-25T12:22:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-40bda-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:31Z" name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd.18b2ce689117e9b8 namespace: kserve-ci-e2e-test resourceVersion: "14536" uid: b11123b9-6d6d-468f-810f-fb93d6c5a5ad reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd namespace: kserve-ci-e2e-test resourceVersion: "14530" uid: 771a87fe-641c-40bd-8588-a6241156f7b0 kind: Event lastTimestamp: "2026-05-25T12:22:32Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:22:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:22:32Z" name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd.18b2ce68c602ba78 namespace: kserve-ci-e2e-test resourceVersion: "14553" uid: 1360c84a-fade-4510-9a12-c1fa92ca0ce0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd namespace: kserve-ci-e2e-test resourceVersion: "14522" uid: 771a87fe-641c-40bd-8588-a6241156f7b0 kind: Event lastTimestamp: "2026-05-25T12:22:32Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T12:22:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:32Z" name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd.18b2ce68c7d5dc2c namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: ea6ca94f-843d-45e7-965d-7fb0abcca867 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd namespace: kserve-ci-e2e-test resourceVersion: "14522" uid: 771a87fe-641c-40bd-8588-a6241156f7b0 kind: Event lastTimestamp: "2026-05-25T12:22:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:22:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:32Z" name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd.18b2ce68ca90c066 namespace: kserve-ci-e2e-test resourceVersion: "14561" uid: d0b9f0bf-ed97-477f-95ef-5cf3e0941a6d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd namespace: kserve-ci-e2e-test resourceVersion: "14522" uid: 771a87fe-641c-40bd-8588-a6241156f7b0 kind: Event lastTimestamp: "2026-05-25T12:22:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:22:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:32Z" name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd.18b2ce68caffa634 namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: dc09dd53-541b-4409-ad28-a4a71ab6868f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd namespace: kserve-ci-e2e-test resourceVersion: "14522" uid: 771a87fe-641c-40bd-8588-a6241156f7b0 kind: Event lastTimestamp: "2026-05-25T12:22:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T12:22:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:32Z" name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd.18b2ce68cb0bcdc9 namespace: kserve-ci-e2e-test resourceVersion: "14563" uid: 5c8bd13b-eb93-4c5e-a28f-b3c62aa806dc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd namespace: kserve-ci-e2e-test resourceVersion: "14522" uid: 771a87fe-641c-40bd-8588-a6241156f7b0 kind: Event lastTimestamp: "2026-05-25T12:22:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:22:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:32Z" name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd.18b2ce68ce5f6402 namespace: kserve-ci-e2e-test resourceVersion: "14565" uid: fff41fe4-275a-4c16-b7fd-b18a06a6f664 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd namespace: kserve-ci-e2e-test resourceVersion: "14522" uid: 771a87fe-641c-40bd-8588-a6241156f7b0 kind: Event lastTimestamp: "2026-05-25T12:22:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:22:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:32Z" name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd.18b2ce68ced81c83 namespace: kserve-ci-e2e-test resourceVersion: "14566" uid: d0619f4a-fe35-43f3-b36e-5ddc7941b4d2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:22:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd namespace: kserve-ci-e2e-test resourceVersion: "14522" uid: 771a87fe-641c-40bd-8588-a6241156f7b0 kind: Event lastTimestamp: "2026-05-25T12:23:09Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:22:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:09Z" name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd.18b2ce6930f00ca4 namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: 6d735722-479e-4ad2-a20a-d415e1360934 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd namespace: kserve-ci-e2e-test resourceVersion: "14522" uid: 771a87fe-641c-40bd-8588-a6241156f7b0 kind: Event lastTimestamp: "2026-05-25T12:31:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:31:46Z" name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd.18b2cee9b93ba77c namespace: kserve-ci-e2e-test resourceVersion: "18878" uid: ed035e89-6ff3-44a0-9722-4a37fcc210aa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd namespace: kserve-ci-e2e-test resourceVersion: "14522" uid: 771a87fe-641c-40bd-8588-a6241156f7b0 kind: Event lastTimestamp: "2026-05-25T12:31:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:31:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:31:45Z" name: error-404-isvc-40bda-predictor-6664b4979f-g8jxd.18b2cee9b93c0c51 namespace: kserve-ci-e2e-test resourceVersion: "18875" uid: 24aa9095-cdf8-4bfe-a77d-da0a2ae50340 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-40bda-predictor-6664b4979f namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: d67f0227-75b1-4519-883c-ee5e13e9b9ac kind: Event lastTimestamp: "2026-05-25T12:22:31Z" message: 'Created pod: error-404-isvc-40bda-predictor-6664b4979f-g8jxd' metadata: creationTimestamp: "2026-05-25T12:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:31Z" name: error-404-isvc-40bda-predictor-6664b4979f.18b2ce6887ae0208 namespace: kserve-ci-e2e-test resourceVersion: "14521" uid: b9a0de21-417b-48e0-86f0-d4264930c947 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-40bda-predictor namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: 148bddbf-5036-4519-b50a-12003fe31297 kind: Event lastTimestamp: "2026-05-25T12:22:31Z" message: Scaled up replica set error-404-isvc-40bda-predictor-6664b4979f from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:31Z" name: error-404-isvc-40bda-predictor.18b2ce6886200743 namespace: kserve-ci-e2e-test resourceVersion: "14516" uid: ed398738-5deb-4b40-934c-c8a3d705428e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:22:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-40bda-predictor namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 95eaf10b-cefe-4221-a123-ca5eb7218371 kind: Event lastTimestamp: "2026-05-25T12:23:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:22:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:23:16Z" name: error-404-isvc-40bda-predictor.18b2ce6c04af6025 namespace: kserve-ci-e2e-test resourceVersion: "15168" uid: 3542cab5-c148-4379-a3ff-2d1c15bc418e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:22:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-40bda-predictor namespace: kserve-ci-e2e-test resourceVersion: "14513" uid: 95eaf10b-cefe-4221-a123-ca5eb7218371 kind: Event lastTimestamp: "2026-05-25T12:23:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:22:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:23:16Z" name: error-404-isvc-40bda-predictor.18b2ce6c05274483 namespace: kserve-ci-e2e-test resourceVersion: "15171" uid: 684e4f2b-b089-4a0a-a180-0c456e218866 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:23:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-40bda-predictor namespace: kserve-ci-e2e-test resourceVersion: "14700" uid: 95eaf10b-cefe-4221-a123-ca5eb7218371 kind: Event lastTimestamp: "2026-05-25T12:23:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:23:46Z" name: error-404-isvc-40bda-predictor.18b2ce6f83fa8c14 namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: fb16b5b5-bb40-4597-b317-cfede806d9d4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:23:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-40bda-predictor namespace: kserve-ci-e2e-test resourceVersion: "14700" uid: 95eaf10b-cefe-4221-a123-ca5eb7218371 kind: Event lastTimestamp: "2026-05-25T12:23:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:23:46Z" name: error-404-isvc-40bda-predictor.18b2ce6f8467cdeb namespace: kserve-ci-e2e-test resourceVersion: "15464" uid: abbb6785-5c38-41a8-baff-75cecb1cb06f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:22:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-40bda namespace: kserve-ci-e2e-test resourceVersion: "14498" uid: e0fae8e4-3d1a-480a-a7f0-ca7ffe0c05df kind: Event lastTimestamp: "2026-05-25T12:23:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:23:20Z" name: error-404-isvc-40bda.18b2ce6885fd870e namespace: kserve-ci-e2e-test resourceVersion: "15218" uid: d9c4b36b-faeb-4ac4-b534-95fb2a267e2d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-40bda namespace: kserve-ci-e2e-test resourceVersion: "15215" uid: e0fae8e4-3d1a-480a-a7f0-ca7ffe0c05df kind: Event lastTimestamp: "2026-05-25T12:23:20Z" message: InferenceService [error-404-isvc-40bda] is Ready metadata: creationTimestamp: "2026-05-25T12:23:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:23:20Z" name: error-404-isvc-40bda.18b2ce73eca349d4 namespace: kserve-ci-e2e-test resourceVersion: "15216" uid: f6b4e51b-41a2-4804-90bc-7b94f94d2e9a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:33:36.441087Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5 namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: d0a4c463-e906-4147-b266-b3d9b2fd6c29 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5 to ip-10-0-142-53.ec2.internal metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:33:36Z" name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5.18b2cf037052c6b7 namespace: kserve-ci-e2e-test resourceVersion: "20338" uid: cb3c31fb-2122-4840-b263-6cf65da6bd69 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5 namespace: kserve-ci-e2e-test resourceVersion: "20343" uid: d0a4c463-e906-4147-b266-b3d9b2fd6c29 kind: Event lastTimestamp: "2026-05-25T12:33:36Z" message: Add eth0 [10.134.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:33:36Z" name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5.18b2cf038aa58896 namespace: kserve-ci-e2e-test resourceVersion: "20354" uid: dfceed77-f48f-4474-8168-598433a3c044 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5 namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: d0a4c463-e906-4147-b266-b3d9b2fd6c29 kind: Event lastTimestamp: "2026-05-25T12:33:36Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:36Z" name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5.18b2cf038c5dddcb namespace: kserve-ci-e2e-test resourceVersion: "20356" uid: 3e298907-6749-4869-95a8-67c13db30cda reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5 namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: d0a4c463-e906-4147-b266-b3d9b2fd6c29 kind: Event lastTimestamp: "2026-05-25T12:33:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:36Z" name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5.18b2cf038f701e47 namespace: kserve-ci-e2e-test resourceVersion: "20357" uid: 6be3c835-e55b-4bd4-9469-e16d5b71c68f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5 namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: d0a4c463-e906-4147-b266-b3d9b2fd6c29 kind: Event lastTimestamp: "2026-05-25T12:33:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:36Z" name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5.18b2cf038fedcf1b namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: 78fa18f2-dfda-463b-aa29-c9284297f1fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5 namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: d0a4c463-e906-4147-b266-b3d9b2fd6c29 kind: Event lastTimestamp: "2026-05-25T12:33:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:36Z" name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5.18b2cf038ff96d20 namespace: kserve-ci-e2e-test resourceVersion: "20359" uid: 2f7be822-e6f1-42a5-b1c6-cd62b5a4b3be reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5 namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: d0a4c463-e906-4147-b266-b3d9b2fd6c29 kind: Event lastTimestamp: "2026-05-25T12:33:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:37Z" name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5.18b2cf039300c47e namespace: kserve-ci-e2e-test resourceVersion: "20361" uid: aeea44cb-9086-4424-8ede-82e919e064c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5 namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: d0a4c463-e906-4147-b266-b3d9b2fd6c29 kind: Event lastTimestamp: "2026-05-25T12:33:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:37Z" name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5.18b2cf03938690c0 namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: 708c2e28-5d06-4c99-8fc3-c390cb865e68 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T12:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5 namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: d0a4c463-e906-4147-b266-b3d9b2fd6c29 kind: Event lastTimestamp: "2026-05-25T12:42:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:54Z" name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5.18b2cf03e73d4dcd namespace: kserve-ci-e2e-test resourceVersion: "24688" uid: 31677676-1ec8-44b9-af66-cd76aaf9d188 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5 namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: d0a4c463-e906-4147-b266-b3d9b2fd6c29 kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:51Z" name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5.18b2cf849bfaa4d8 namespace: kserve-ci-e2e-test resourceVersion: "24596" uid: a78e9c79-4634-4ad9-8243-e291081a531c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5 namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: d0a4c463-e906-4147-b266-b3d9b2fd6c29 kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:51Z" name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5.18b2cf849bfb6418 namespace: kserve-ci-e2e-test resourceVersion: "24598" uid: 9902d485-8f27-4e1f-a428-e8157b1c2535 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5 namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: d0a4c463-e906-4147-b266-b3d9b2fd6c29 kind: Event lastTimestamp: "2026-05-25T12:42:54Z" message: 'Readiness probe failed: Get "https://10.134.0.47:8643/healthz": dial tcp 10.134.0.47:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:42:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:54Z" name: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5.18b2cf855b965620 namespace: kserve-ci-e2e-test resourceVersion: "24687" uid: 25e794d1-4783-43e6-a245-de4b03a9abc8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4d75e-predictor-5c8bb955c5 namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: bc93c845-b86a-44f1-9713-ac209ffdc609 kind: Event lastTimestamp: "2026-05-25T12:33:36Z" message: 'Created pod: error-404-isvc-4d75e-predictor-5c8bb955c5-mqnb5' metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:33:36Z" name: error-404-isvc-4d75e-predictor-5c8bb955c5.18b2cf036f9bc845 namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: 6d545d70-25ca-4331-b0d7-09290976d069 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4d75e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: d057257c-540b-4b60-89ce-37d8ab80665d kind: Event lastTimestamp: "2026-05-25T12:33:36Z" message: Scaled up replica set error-404-isvc-4d75e-predictor-5c8bb955c5 from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:33:36Z" name: error-404-isvc-4d75e-predictor.18b2cf036d9027c4 namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 703ed5ab-43e6-467c-aced-3e7d8a37f993 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:33:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4d75e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20326" uid: 06cee031-e2f8-4aeb-94ad-29bc34065631 kind: Event lastTimestamp: "2026-05-25T12:34:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:34:06Z" name: error-404-isvc-4d75e-predictor.18b2cf06ec2c2011 namespace: kserve-ci-e2e-test resourceVersion: "20657" uid: 09a49c99-cfeb-4ba6-99c2-3cf756264cc4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:33:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4d75e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20326" uid: 06cee031-e2f8-4aeb-94ad-29bc34065631 kind: Event lastTimestamp: "2026-05-25T12:34:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:34:06Z" name: error-404-isvc-4d75e-predictor.18b2cf06ec84ed36 namespace: kserve-ci-e2e-test resourceVersion: "20660" uid: c64ebb5c-a9c2-4f59-ae2f-845adc7fe74e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:34:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4d75e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20508" uid: 06cee031-e2f8-4aeb-94ad-29bc34065631 kind: Event lastTimestamp: "2026-05-25T12:34:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:34:51Z" name: error-404-isvc-4d75e-predictor.18b2cf0dea75c89a namespace: kserve-ci-e2e-test resourceVersion: "21086" uid: 4efa6c71-b471-4cfb-aeef-164620e6135d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:34:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4d75e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20508" uid: 06cee031-e2f8-4aeb-94ad-29bc34065631 kind: Event lastTimestamp: "2026-05-25T12:34:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:34:51Z" name: error-404-isvc-4d75e-predictor.18b2cf0dead23b21 namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: f5e39ea6-6acb-4781-9455-578fa462f905 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4d75e namespace: kserve-ci-e2e-test resourceVersion: "20314" uid: 714ee519-3992-4f84-8a25-42a2e4c78350 kind: Event lastTimestamp: "2026-05-25T12:34:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:34:24Z" name: error-404-isvc-4d75e.18b2cf036d6dd3ac namespace: kserve-ci-e2e-test resourceVersion: "20837" uid: 359a8cf8-4632-45f4-934e-e4f46acad1a6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4d75e namespace: kserve-ci-e2e-test resourceVersion: "20314" uid: 714ee519-3992-4f84-8a25-42a2e4c78350 kind: Event lastTimestamp: "2026-05-25T12:33:36Z" message: 'Failed to update status for InferenceService "error-404-isvc-4d75e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4d75e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:33:36Z" name: error-404-isvc-4d75e.18b2cf036e4eecdb namespace: kserve-ci-e2e-test resourceVersion: "20331" uid: e84ad204-4114-429d-9572-d7426afaf10e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4d75e namespace: kserve-ci-e2e-test resourceVersion: "20314" uid: 714ee519-3992-4f84-8a25-42a2e4c78350 kind: Event lastTimestamp: "2026-05-25T12:33:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4d75e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:33:36Z" name: error-404-isvc-4d75e.18b2cf036e4f4182 namespace: kserve-ci-e2e-test resourceVersion: "20333" uid: 2eb79188-c7fe-4a05-9b71-5ddad1d22e96 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:34:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4d75e namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: 714ee519-3992-4f84-8a25-42a2e4c78350 kind: Event lastTimestamp: "2026-05-25T12:34:24Z" message: InferenceService [error-404-isvc-4d75e] is Ready metadata: creationTimestamp: "2026-05-25T12:34:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:34:24Z" name: error-404-isvc-4d75e.18b2cf0ea48b98c1 namespace: kserve-ci-e2e-test resourceVersion: "20836" uid: 45adff28-9424-4665-ac72-8e831524307b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:23:01.160044Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8 namespace: kserve-ci-e2e-test resourceVersion: "14983" uid: 40582ba8-ab3b-4046-985c-bcfb29eb0b82 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8 to ip-10-0-142-53.ec2.internal metadata: creationTimestamp: "2026-05-25T12:23:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:23:01Z" name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8.18b2ce6f869f5a68 namespace: kserve-ci-e2e-test resourceVersion: "14988" uid: 2e55b0dc-6326-4769-9be5-1c1d09a37076 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8 namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: 40582ba8-ab3b-4046-985c-bcfb29eb0b82 kind: Event lastTimestamp: "2026-05-25T12:23:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-833d6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:01Z" name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8.18b2ce6f91cf65cf namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: 6587c79c-a142-4848-b5e4-bc0b3138dda0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8 namespace: kserve-ci-e2e-test resourceVersion: "14994" uid: 40582ba8-ab3b-4046-985c-bcfb29eb0b82 kind: Event lastTimestamp: "2026-05-25T12:23:02Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:23:02Z" name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8.18b2ce6fc4cc1a73 namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: f4586a8c-598e-4158-b804-1579a6609621 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8 namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: 40582ba8-ab3b-4046-985c-bcfb29eb0b82 kind: Event lastTimestamp: "2026-05-25T12:23:02Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T12:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:02Z" name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8.18b2ce6fc6631377 namespace: kserve-ci-e2e-test resourceVersion: "15019" uid: e49228f9-ef0c-4fe4-ab76-f521afc399c4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8 namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: 40582ba8-ab3b-4046-985c-bcfb29eb0b82 kind: Event lastTimestamp: "2026-05-25T12:23:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:02Z" name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8.18b2ce6fc9a35cb1 namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: 6038161c-9633-4ce2-83eb-7feaf3397adb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8 namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: 40582ba8-ab3b-4046-985c-bcfb29eb0b82 kind: Event lastTimestamp: "2026-05-25T12:23:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:02Z" name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8.18b2ce6fca3c9fd6 namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: 9c466e3f-7f21-4b5c-a309-2e60cec64ac8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8 namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: 40582ba8-ab3b-4046-985c-bcfb29eb0b82 kind: Event lastTimestamp: "2026-05-25T12:23:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T12:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:02Z" name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8.18b2ce6fca4a63b6 namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: dd282c92-0d2a-48fc-b720-19c36d303b88 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8 namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: 40582ba8-ab3b-4046-985c-bcfb29eb0b82 kind: Event lastTimestamp: "2026-05-25T12:23:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:02Z" name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8.18b2ce6fcd81ae72 namespace: kserve-ci-e2e-test resourceVersion: "15026" uid: 4298bd74-adef-4432-84c8-a8655ec9aaf9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8 namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: 40582ba8-ab3b-4046-985c-bcfb29eb0b82 kind: Event lastTimestamp: "2026-05-25T12:23:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:02Z" name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8.18b2ce6fcdf78595 namespace: kserve-ci-e2e-test resourceVersion: "15027" uid: 251d47bf-bbe4-48c3-924c-d63b138a8c47 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:23:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8 namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: 40582ba8-ab3b-4046-985c-bcfb29eb0b82 kind: Event lastTimestamp: "2026-05-25T12:23:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:40Z" name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8.18b2ce70341a11c7 namespace: kserve-ci-e2e-test resourceVersion: "15413" uid: 14e31804-10fd-4ff2-9c40-ba79d669e228 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8 namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: 40582ba8-ab3b-4046-985c-bcfb29eb0b82 kind: Event lastTimestamp: "2026-05-25T12:32:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:16Z" name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8.18b2cef0bcd7c151 namespace: kserve-ci-e2e-test resourceVersion: "19274" uid: 067315bb-5659-4c9c-a6df-d9ee74853ff3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8 namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: 40582ba8-ab3b-4046-985c-bcfb29eb0b82 kind: Event lastTimestamp: "2026-05-25T12:32:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:16Z" name: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8.18b2cef0bcd83dc1 namespace: kserve-ci-e2e-test resourceVersion: "19278" uid: e9cb3372-8073-43fb-bd19-03849a2c3bf8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-833d6-predictor-7dfb76b5f7 namespace: kserve-ci-e2e-test resourceVersion: "14975" uid: 89fad2ee-9e55-418f-acf1-b8e30bfef9a5 kind: Event lastTimestamp: "2026-05-25T12:23:01Z" message: 'Created pod: error-404-isvc-833d6-predictor-7dfb76b5f7-8rzb8' metadata: creationTimestamp: "2026-05-25T12:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:23:01Z" name: error-404-isvc-833d6-predictor-7dfb76b5f7.18b2ce6f863071c9 namespace: kserve-ci-e2e-test resourceVersion: "14985" uid: 221725ec-585a-488d-8288-5caddaddebcc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-833d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14971" uid: 87be5e99-994a-4023-b95e-635f2df3d32d kind: Event lastTimestamp: "2026-05-25T12:23:01Z" message: Scaled up replica set error-404-isvc-833d6-predictor-7dfb76b5f7 from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:23:01Z" name: error-404-isvc-833d6-predictor.18b2ce6f84a82594 namespace: kserve-ci-e2e-test resourceVersion: "14978" uid: 4b0b631a-0035-4edb-b8f1-a6d824bbbe87 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-833d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: ef8b0c0d-7e00-470e-99dd-6ec844330301 kind: Event lastTimestamp: "2026-05-25T12:23:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:23:16Z" name: error-404-isvc-833d6-predictor.18b2ce7303b83e65 namespace: kserve-ci-e2e-test resourceVersion: "15169" uid: b48ae52f-03b3-495b-80de-b17a1db67210 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-833d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14974" uid: ef8b0c0d-7e00-470e-99dd-6ec844330301 kind: Event lastTimestamp: "2026-05-25T12:23:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:23:16Z" name: error-404-isvc-833d6-predictor.18b2ce7304434e94 namespace: kserve-ci-e2e-test resourceVersion: "15172" uid: e215c917-0b17-45e7-8eb4-b076b5dcaa37 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:23:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-833d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: ef8b0c0d-7e00-470e-99dd-6ec844330301 kind: Event lastTimestamp: "2026-05-25T12:24:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:23:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:24:31Z" name: error-404-isvc-833d6-predictor.18b2ce76831b2b40 namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 861a9574-a6a8-49ff-812d-8bc8fb7a1d5d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:23:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-833d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: ef8b0c0d-7e00-470e-99dd-6ec844330301 kind: Event lastTimestamp: "2026-05-25T12:24:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:23:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:24:31Z" name: error-404-isvc-833d6-predictor.18b2ce768388966a namespace: kserve-ci-e2e-test resourceVersion: "15864" uid: 46584c8b-9acc-4b2b-9cd9-9b05df81a895 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:23:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-833d6 namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: cc63b6de-e850-49e9-94d3-616a6901c53e kind: Event lastTimestamp: "2026-05-25T12:23:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:23:50Z" name: error-404-isvc-833d6.18b2ce6f84803093 namespace: kserve-ci-e2e-test resourceVersion: "15510" uid: a7b43b10-a8bb-4fe4-8460-35cc8cdf038e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-833d6 namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: cc63b6de-e850-49e9-94d3-616a6901c53e kind: Event lastTimestamp: "2026-05-25T12:23:01Z" message: 'Failed to update status for InferenceService "error-404-isvc-833d6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-833d6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:23:01Z" name: error-404-isvc-833d6.18b2ce6f84d96a19 namespace: kserve-ci-e2e-test resourceVersion: "14980" uid: 4ff780d9-74ec-4892-a5b0-3f2c7295bd9f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-833d6 namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: cc63b6de-e850-49e9-94d3-616a6901c53e kind: Event lastTimestamp: "2026-05-25T12:23:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-833d6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:23:01Z" name: error-404-isvc-833d6.18b2ce6f84d9cc31 namespace: kserve-ci-e2e-test resourceVersion: "14981" uid: 2a163a8a-d7ce-4887-a774-68287e53c472 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-833d6 namespace: kserve-ci-e2e-test resourceVersion: "15508" uid: cc63b6de-e850-49e9-94d3-616a6901c53e kind: Event lastTimestamp: "2026-05-25T12:23:50Z" message: InferenceService [error-404-isvc-833d6] is Ready metadata: creationTimestamp: "2026-05-25T12:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:23:50Z" name: error-404-isvc-833d6.18b2ce7aeff2b2ca namespace: kserve-ci-e2e-test resourceVersion: "15509" uid: 8203d038-fafe-4158-95cb-411db8251bd3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:43:31.706796Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh namespace: kserve-ci-e2e-test resourceVersion: "25137" uid: 8caf44b4-889f-4d31-b6d0-1ae3beafd3b7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8902c-predictor-86c999b5df-f6nxh to ip-10-0-142-53.ec2.internal metadata: creationTimestamp: "2026-05-25T12:43:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:43:31Z" name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh.18b2cf8e08ecae35 namespace: kserve-ci-e2e-test resourceVersion: "25141" uid: 01c53d07-3171-42fa-b95a-ceb3238b48c2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh namespace: kserve-ci-e2e-test resourceVersion: "25139" uid: 8caf44b4-889f-4d31-b6d0-1ae3beafd3b7 kind: Event lastTimestamp: "2026-05-25T12:43:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-8902c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:31Z" name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh.18b2cf8e133eb2ad namespace: kserve-ci-e2e-test resourceVersion: "25154" uid: 0e0999ab-71af-49e7-a840-006c19b53940 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh namespace: kserve-ci-e2e-test resourceVersion: "25146" uid: 8caf44b4-889f-4d31-b6d0-1ae3beafd3b7 kind: Event lastTimestamp: "2026-05-25T12:43:32Z" message: Add eth0 [10.134.0.53/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:43:32Z" name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh.18b2cf8e47714b94 namespace: kserve-ci-e2e-test resourceVersion: "25176" uid: e7967fcb-bbbd-4f85-a6e1-3c1105a3f2d2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh namespace: kserve-ci-e2e-test resourceVersion: "25139" uid: 8caf44b4-889f-4d31-b6d0-1ae3beafd3b7 kind: Event lastTimestamp: "2026-05-25T12:43:32Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T12:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:32Z" name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh.18b2cf8e48fca945 namespace: kserve-ci-e2e-test resourceVersion: "25178" uid: 8fd0cef9-1284-418e-9ad6-b2e5f27a4ada reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh namespace: kserve-ci-e2e-test resourceVersion: "25139" uid: 8caf44b4-889f-4d31-b6d0-1ae3beafd3b7 kind: Event lastTimestamp: "2026-05-25T12:43:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:32Z" name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh.18b2cf8e4bea5d1b namespace: kserve-ci-e2e-test resourceVersion: "25181" uid: 5691d11e-7413-4604-8b97-ff50a168a9ff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh namespace: kserve-ci-e2e-test resourceVersion: "25139" uid: 8caf44b4-889f-4d31-b6d0-1ae3beafd3b7 kind: Event lastTimestamp: "2026-05-25T12:43:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:32Z" name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh.18b2cf8e4c793280 namespace: kserve-ci-e2e-test resourceVersion: "25182" uid: 5052ad93-dd8f-4922-bdc7-763855718315 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh namespace: kserve-ci-e2e-test resourceVersion: "25139" uid: 8caf44b4-889f-4d31-b6d0-1ae3beafd3b7 kind: Event lastTimestamp: "2026-05-25T12:43:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T12:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:32Z" name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh.18b2cf8e4c8594ba namespace: kserve-ci-e2e-test resourceVersion: "25183" uid: b1662304-3643-4145-a8dc-37712eec0c9c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh namespace: kserve-ci-e2e-test resourceVersion: "25139" uid: 8caf44b4-889f-4d31-b6d0-1ae3beafd3b7 kind: Event lastTimestamp: "2026-05-25T12:43:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:32Z" name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh.18b2cf8e4fb3ebba namespace: kserve-ci-e2e-test resourceVersion: "25184" uid: 262bc843-8c40-4fdf-88bb-d34ee72637cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh namespace: kserve-ci-e2e-test resourceVersion: "25139" uid: 8caf44b4-889f-4d31-b6d0-1ae3beafd3b7 kind: Event lastTimestamp: "2026-05-25T12:43:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:32Z" name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh.18b2cf8e5034936f namespace: kserve-ci-e2e-test resourceVersion: "25185" uid: 6f1c2a61-5dba-4253-8836-bb16900b3470 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T12:43:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh namespace: kserve-ci-e2e-test resourceVersion: "25139" uid: 8caf44b4-889f-4d31-b6d0-1ae3beafd3b7 kind: Event lastTimestamp: "2026-05-25T12:44:10Z" message: 'Readiness probe failed: dial tcp 10.134.0.53:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:44:10Z" name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh.18b2cf8e7efd8dfb namespace: kserve-ci-e2e-test resourceVersion: "25567" uid: 4616fcf6-0f55-4772-b89b-258f9b3895a8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:52:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh namespace: kserve-ci-e2e-test resourceVersion: "25139" uid: 8caf44b4-889f-4d31-b6d0-1ae3beafd3b7 kind: Event lastTimestamp: "2026-05-25T12:52:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:52:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:52:46Z" name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh.18b2d00f3bff0924 namespace: kserve-ci-e2e-test resourceVersion: "29536" uid: 5c84ea9c-988f-481c-b876-3945df86fb91 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:52:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh namespace: kserve-ci-e2e-test resourceVersion: "25139" uid: 8caf44b4-889f-4d31-b6d0-1ae3beafd3b7 kind: Event lastTimestamp: "2026-05-25T12:52:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:52:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:52:46Z" name: error-404-isvc-8902c-predictor-86c999b5df-f6nxh.18b2d00f3bff76d4 namespace: kserve-ci-e2e-test resourceVersion: "29537" uid: b2b421ed-c263-4ea2-b2e9-8dae6129568a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8902c-predictor-86c999b5df namespace: kserve-ci-e2e-test resourceVersion: "25130" uid: 56fb2b4b-e45d-4c95-b469-0cd5ebb57711 kind: Event lastTimestamp: "2026-05-25T12:43:31Z" message: 'Created pod: error-404-isvc-8902c-predictor-86c999b5df-f6nxh' metadata: creationTimestamp: "2026-05-25T12:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:43:31Z" name: error-404-isvc-8902c-predictor-86c999b5df.18b2cf8e087777f4 namespace: kserve-ci-e2e-test resourceVersion: "25138" uid: cafbfea4-4363-461d-b3e0-d93debb3857f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8902c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25128" uid: 0935ce6c-ad27-437b-9562-6e8bb08138d9 kind: Event lastTimestamp: "2026-05-25T12:43:31Z" message: Scaled up replica set error-404-isvc-8902c-predictor-86c999b5df from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:43:31Z" name: error-404-isvc-8902c-predictor.18b2cf8e05fd5f38 namespace: kserve-ci-e2e-test resourceVersion: "25132" uid: 1ba9e0af-85ad-417c-9036-538c6b589ab2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:43:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8902c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25129" uid: 04821cc4-c5cb-421f-acef-3e32d6c6779f kind: Event lastTimestamp: "2026-05-25T12:44:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:44:31Z" name: error-404-isvc-8902c-predictor.18b2cf9184806532 namespace: kserve-ci-e2e-test resourceVersion: "25772" uid: 8b888224-1d4f-4708-befa-6586df6c4faf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:43:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8902c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25129" uid: 04821cc4-c5cb-421f-acef-3e32d6c6779f kind: Event lastTimestamp: "2026-05-25T12:44:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:44:31Z" name: error-404-isvc-8902c-predictor.18b2cf9184f04751 namespace: kserve-ci-e2e-test resourceVersion: "25774" uid: ae16a88d-a87e-451f-98db-114cda38ee88 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8902c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25325" uid: 04821cc4-c5cb-421f-acef-3e32d6c6779f kind: Event lastTimestamp: "2026-05-25T12:44:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:44:01Z" name: error-404-isvc-8902c-predictor.18b2cf95039ff2bd namespace: kserve-ci-e2e-test resourceVersion: "25480" uid: 08e89100-12f7-4181-9fb7-b1dd37233cf7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8902c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25325" uid: 04821cc4-c5cb-421f-acef-3e32d6c6779f kind: Event lastTimestamp: "2026-05-25T12:44:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:44:01Z" name: error-404-isvc-8902c-predictor.18b2cf9504117687 namespace: kserve-ci-e2e-test resourceVersion: "25483" uid: 68372a6f-4a61-4829-8561-147e9926c094 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:43:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8902c namespace: kserve-ci-e2e-test resourceVersion: "25114" uid: 9f6a23bf-f5c8-4ea6-88cb-9d965a92f5ba kind: Event lastTimestamp: "2026-05-25T12:44:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:44:20Z" name: error-404-isvc-8902c.18b2cf8e05d39568 namespace: kserve-ci-e2e-test resourceVersion: "25662" uid: 5c15566b-7cf6-4552-af64-015b4f903596 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8902c namespace: kserve-ci-e2e-test resourceVersion: "25114" uid: 9f6a23bf-f5c8-4ea6-88cb-9d965a92f5ba kind: Event lastTimestamp: "2026-05-25T12:43:31Z" message: 'Failed to update status for InferenceService "error-404-isvc-8902c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8902c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:43:31Z" name: error-404-isvc-8902c.18b2cf8e06770b91 namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: 7abaf31e-def5-419a-bcd0-23cf405589ba reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8902c namespace: kserve-ci-e2e-test resourceVersion: "25114" uid: 9f6a23bf-f5c8-4ea6-88cb-9d965a92f5ba kind: Event lastTimestamp: "2026-05-25T12:43:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8902c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:43:31Z" name: error-404-isvc-8902c.18b2cf8e06777e4d namespace: kserve-ci-e2e-test resourceVersion: "25135" uid: 2f184fdb-f142-407d-a14c-6668891cd77b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8902c namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: 9f6a23bf-f5c8-4ea6-88cb-9d965a92f5ba kind: Event lastTimestamp: "2026-05-25T12:44:20Z" message: InferenceService [error-404-isvc-8902c] is Ready metadata: creationTimestamp: "2026-05-25T12:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:44:20Z" name: error-404-isvc-8902c.18b2cf997a1051e9 namespace: kserve-ci-e2e-test resourceVersion: "25660" uid: a528321c-2132-459e-842f-a4aa0f9aefac reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:32:16.228926Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-acbca-predictor-687469f549-r5cwv namespace: kserve-ci-e2e-test resourceVersion: "19284" uid: 51175389-4295-493b-82bc-812c2e34c778 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-acbca-predictor-687469f549-r5cwv to ip-10-0-142-53.ec2.internal metadata: creationTimestamp: "2026-05-25T12:32:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:32:16Z" name: error-404-isvc-acbca-predictor-687469f549-r5cwv.18b2cef0c34e55cd namespace: kserve-ci-e2e-test resourceVersion: "19288" uid: 83ce4911-902e-4034-bdb3-678e9f799aab reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-acbca-predictor-687469f549-r5cwv namespace: kserve-ci-e2e-test resourceVersion: "19285" uid: 51175389-4295-493b-82bc-812c2e34c778 kind: Event lastTimestamp: "2026-05-25T12:32:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-acbca-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:16Z" name: error-404-isvc-acbca-predictor-687469f549-r5cwv.18b2cef0cb7fc256 namespace: kserve-ci-e2e-test resourceVersion: "19299" uid: 19a246d1-af52-42fc-ab71-405d492816ae reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-acbca-predictor-687469f549-r5cwv namespace: kserve-ci-e2e-test resourceVersion: "19296" uid: 51175389-4295-493b-82bc-812c2e34c778 kind: Event lastTimestamp: "2026-05-25T12:32:17Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:32:17Z" name: error-404-isvc-acbca-predictor-687469f549-r5cwv.18b2cef101575d25 namespace: kserve-ci-e2e-test resourceVersion: "19319" uid: fd5de968-8e97-4dd3-a73f-773966f1294a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-acbca-predictor-687469f549-r5cwv namespace: kserve-ci-e2e-test resourceVersion: "19285" uid: 51175389-4295-493b-82bc-812c2e34c778 kind: Event lastTimestamp: "2026-05-25T12:32:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T12:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:17Z" name: error-404-isvc-acbca-predictor-687469f549-r5cwv.18b2cef1032bd2d5 namespace: kserve-ci-e2e-test resourceVersion: "19321" uid: 4b1f1dd4-f2f2-4960-b8ee-bab442174659 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-acbca-predictor-687469f549-r5cwv namespace: kserve-ci-e2e-test resourceVersion: "19285" uid: 51175389-4295-493b-82bc-812c2e34c778 kind: Event lastTimestamp: "2026-05-25T12:32:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:17Z" name: error-404-isvc-acbca-predictor-687469f549-r5cwv.18b2cef105e8b79c namespace: kserve-ci-e2e-test resourceVersion: "19322" uid: 40501512-2f07-4967-b364-cddf33ce0586 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-acbca-predictor-687469f549-r5cwv namespace: kserve-ci-e2e-test resourceVersion: "19285" uid: 51175389-4295-493b-82bc-812c2e34c778 kind: Event lastTimestamp: "2026-05-25T12:32:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:17Z" name: error-404-isvc-acbca-predictor-687469f549-r5cwv.18b2cef10655c21a namespace: kserve-ci-e2e-test resourceVersion: "19323" uid: 98796dba-6d2b-41dc-901d-c2d7aee25045 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-acbca-predictor-687469f549-r5cwv namespace: kserve-ci-e2e-test resourceVersion: "19285" uid: 51175389-4295-493b-82bc-812c2e34c778 kind: Event lastTimestamp: "2026-05-25T12:32:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T12:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:17Z" name: error-404-isvc-acbca-predictor-687469f549-r5cwv.18b2cef106627e89 namespace: kserve-ci-e2e-test resourceVersion: "19324" uid: 5d360dc3-e795-4a83-84d6-dd39a7bbff3f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-acbca-predictor-687469f549-r5cwv namespace: kserve-ci-e2e-test resourceVersion: "19285" uid: 51175389-4295-493b-82bc-812c2e34c778 kind: Event lastTimestamp: "2026-05-25T12:32:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:17Z" name: error-404-isvc-acbca-predictor-687469f549-r5cwv.18b2cef10a2f9751 namespace: kserve-ci-e2e-test resourceVersion: "19325" uid: ab461c66-d6a4-4874-90ca-786885d34be0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-acbca-predictor-687469f549-r5cwv namespace: kserve-ci-e2e-test resourceVersion: "19285" uid: 51175389-4295-493b-82bc-812c2e34c778 kind: Event lastTimestamp: "2026-05-25T12:32:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:17Z" name: error-404-isvc-acbca-predictor-687469f549-r5cwv.18b2cef10ac0d973 namespace: kserve-ci-e2e-test resourceVersion: "19326" uid: 78996456-93c3-48c5-8f42-79d176dd71ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:32:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-acbca-predictor-687469f549-r5cwv namespace: kserve-ci-e2e-test resourceVersion: "19285" uid: 51175389-4295-493b-82bc-812c2e34c778 kind: Event lastTimestamp: "2026-05-25T12:32:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:54Z" name: error-404-isvc-acbca-predictor-687469f549-r5cwv.18b2cef13186b8c5 namespace: kserve-ci-e2e-test resourceVersion: "19702" uid: 10ecf346-ddc9-436d-b6f3-55c62d0a5b2a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-acbca-predictor-687469f549-r5cwv namespace: kserve-ci-e2e-test resourceVersion: "19285" uid: 51175389-4295-493b-82bc-812c2e34c778 kind: Event lastTimestamp: "2026-05-25T12:33:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:36Z" name: error-404-isvc-acbca-predictor-687469f549-r5cwv.18b2cf03621fd3af namespace: kserve-ci-e2e-test resourceVersion: "20277" uid: 6f87cd21-299c-4d6d-bf4a-b90bf9d9f1c4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-acbca-predictor-687469f549-r5cwv namespace: kserve-ci-e2e-test resourceVersion: "19285" uid: 51175389-4295-493b-82bc-812c2e34c778 kind: Event lastTimestamp: "2026-05-25T12:33:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:36Z" name: error-404-isvc-acbca-predictor-687469f549-r5cwv.18b2cf03622081e7 namespace: kserve-ci-e2e-test resourceVersion: "20280" uid: a077a84a-3637-4198-a254-664735edb492 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-acbca-predictor-687469f549-r5cwv namespace: kserve-ci-e2e-test resourceVersion: "19285" uid: 51175389-4295-493b-82bc-812c2e34c778 kind: Event lastTimestamp: "2026-05-25T12:33:39Z" message: 'Readiness probe failed: Get "https://10.134.0.43:8643/healthz": dial tcp 10.134.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:33:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:39Z" name: error-404-isvc-acbca-predictor-687469f549-r5cwv.18b2cf040dccfc50 namespace: kserve-ci-e2e-test resourceVersion: "20400" uid: 64abc966-40e2-46d6-81a9-30e0b8526cd1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-acbca-predictor-687469f549 namespace: kserve-ci-e2e-test resourceVersion: "19272" uid: 746ba568-55ea-4439-8f87-f76a0415a586 kind: Event lastTimestamp: "2026-05-25T12:32:16Z" message: 'Created pod: error-404-isvc-acbca-predictor-687469f549-r5cwv' metadata: creationTimestamp: "2026-05-25T12:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:32:16Z" name: error-404-isvc-acbca-predictor-687469f549.18b2cef0c284e18e namespace: kserve-ci-e2e-test resourceVersion: "19286" uid: 58e3cd05-f33e-4342-9be3-fe70d2aa9f93 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-acbca-predictor namespace: kserve-ci-e2e-test resourceVersion: "19264" uid: caad9273-f605-41ab-bb74-a9cf7bb6efd7 kind: Event lastTimestamp: "2026-05-25T12:32:16Z" message: Scaled up replica set error-404-isvc-acbca-predictor-687469f549 from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:32:16Z" name: error-404-isvc-acbca-predictor.18b2cef0bd9598e7 namespace: kserve-ci-e2e-test resourceVersion: "19276" uid: c9b5f3f7-192c-4d8d-a70c-fb600bd96d6a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:32:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-acbca-predictor namespace: kserve-ci-e2e-test resourceVersion: "19268" uid: 2b6a24b1-3949-4c70-8cc5-ebc78bb44152 kind: Event lastTimestamp: "2026-05-25T12:33:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:33:01Z" name: error-404-isvc-acbca-predictor.18b2cef43b4daedb namespace: kserve-ci-e2e-test resourceVersion: "19901" uid: 578e4545-56e0-4440-b31d-b32aa55498a5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:32:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-acbca-predictor namespace: kserve-ci-e2e-test resourceVersion: "19268" uid: 2b6a24b1-3949-4c70-8cc5-ebc78bb44152 kind: Event lastTimestamp: "2026-05-25T12:33:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:32:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:33:01Z" name: error-404-isvc-acbca-predictor.18b2cef43bc6c8d5 namespace: kserve-ci-e2e-test resourceVersion: "19902" uid: 22da9b61-e7d8-4ab6-a538-5c0f51ee84a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:32:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-acbca-predictor namespace: kserve-ci-e2e-test resourceVersion: "19458" uid: 2b6a24b1-3949-4c70-8cc5-ebc78bb44152 kind: Event lastTimestamp: "2026-05-25T12:33:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:32:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:33:31Z" name: error-404-isvc-acbca-predictor.18b2cef7baafd139 namespace: kserve-ci-e2e-test resourceVersion: "20198" uid: ee4c1b34-4699-47dd-84ad-ba073647cce7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:32:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-acbca-predictor namespace: kserve-ci-e2e-test resourceVersion: "19458" uid: 2b6a24b1-3949-4c70-8cc5-ebc78bb44152 kind: Event lastTimestamp: "2026-05-25T12:33:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:32:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:33:31Z" name: error-404-isvc-acbca-predictor.18b2cef7bb1df7cb namespace: kserve-ci-e2e-test resourceVersion: "20199" uid: 7da42951-d9f3-4a94-b0d2-5a35a83c9f6c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:32:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-acbca namespace: kserve-ci-e2e-test resourceVersion: "19235" uid: 3660b190-7a1c-46e8-8ae4-e45bfa9e3f31 kind: Event lastTimestamp: "2026-05-25T12:33:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:33:04Z" name: error-404-isvc-acbca.18b2cef0bca12b76 namespace: kserve-ci-e2e-test resourceVersion: "19936" uid: c953eec8-b416-4bc2-81e9-58a6e92984c7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-acbca namespace: kserve-ci-e2e-test resourceVersion: "19235" uid: 3660b190-7a1c-46e8-8ae4-e45bfa9e3f31 kind: Event lastTimestamp: "2026-05-25T12:32:16Z" message: 'Failed to update status for InferenceService "error-404-isvc-acbca": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-acbca": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:32:16Z" name: error-404-isvc-acbca.18b2cef0bd1f47ba namespace: kserve-ci-e2e-test resourceVersion: "19275" uid: df02624b-10d9-43fb-85df-2e02dff42ecd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-acbca namespace: kserve-ci-e2e-test resourceVersion: "19235" uid: 3660b190-7a1c-46e8-8ae4-e45bfa9e3f31 kind: Event lastTimestamp: "2026-05-25T12:32:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-acbca": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:32:16Z" name: error-404-isvc-acbca.18b2cef0bd201415 namespace: kserve-ci-e2e-test resourceVersion: "19279" uid: 7c8f79ff-d1a9-47b8-a00e-969e6c99cce4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-acbca namespace: kserve-ci-e2e-test resourceVersion: "19931" uid: 3660b190-7a1c-46e8-8ae4-e45bfa9e3f31 kind: Event lastTimestamp: "2026-05-25T12:33:04Z" message: InferenceService [error-404-isvc-acbca] is Ready metadata: creationTimestamp: "2026-05-25T12:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:33:04Z" name: error-404-isvc-acbca.18b2cefbefcdefcd namespace: kserve-ci-e2e-test resourceVersion: "19933" uid: 4c556478-8047-4473-bb73-b42ddb5c9af5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:31:46.089519Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg namespace: kserve-ci-e2e-test resourceVersion: "18904" uid: 2d880044-04f1-4518-a423-305351e6122d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-bcff4-predictor-55cbf45559-tw9hg to ip-10-0-142-53.ec2.internal metadata: creationTimestamp: "2026-05-25T12:31:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:31:46Z" name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg.18b2cee9bedb8573 namespace: kserve-ci-e2e-test resourceVersion: "18908" uid: 287cbff4-0ac2-42b5-919b-2d4eaedb52a2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg namespace: kserve-ci-e2e-test resourceVersion: "18906" uid: 2d880044-04f1-4518-a423-305351e6122d kind: Event lastTimestamp: "2026-05-25T12:31:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-bcff4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:31:46Z" name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg.18b2cee9cb0eb7b2 namespace: kserve-ci-e2e-test resourceVersion: "18916" uid: 3da07964-22ba-4f08-ae94-b9a4d65db0be reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg namespace: kserve-ci-e2e-test resourceVersion: "18913" uid: 2d880044-04f1-4518-a423-305351e6122d kind: Event lastTimestamp: "2026-05-25T12:31:47Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:31:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:31:47Z" name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg.18b2cee9fc943b0c namespace: kserve-ci-e2e-test resourceVersion: "18941" uid: c970c2d2-6507-49a5-93cf-8db10652d419 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg namespace: kserve-ci-e2e-test resourceVersion: "18906" uid: 2d880044-04f1-4518-a423-305351e6122d kind: Event lastTimestamp: "2026-05-25T12:31:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T12:31:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:31:47Z" name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg.18b2cee9fe6c65b3 namespace: kserve-ci-e2e-test resourceVersion: "18944" uid: 473d1e1d-abf6-4f9e-8778-f66b34da005a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg namespace: kserve-ci-e2e-test resourceVersion: "18906" uid: 2d880044-04f1-4518-a423-305351e6122d kind: Event lastTimestamp: "2026-05-25T12:31:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:31:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:31:47Z" name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg.18b2ceea01f42c77 namespace: kserve-ci-e2e-test resourceVersion: "18945" uid: 389119af-fa53-4e40-a29b-392710b3f2f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg namespace: kserve-ci-e2e-test resourceVersion: "18906" uid: 2d880044-04f1-4518-a423-305351e6122d kind: Event lastTimestamp: "2026-05-25T12:31:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:31:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:31:47Z" name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg.18b2ceea0286c290 namespace: kserve-ci-e2e-test resourceVersion: "18946" uid: ada1aac1-9e4f-4326-858e-88116631cf3b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg namespace: kserve-ci-e2e-test resourceVersion: "18906" uid: 2d880044-04f1-4518-a423-305351e6122d kind: Event lastTimestamp: "2026-05-25T12:31:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T12:31:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:31:47Z" name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg.18b2ceea029314cb namespace: kserve-ci-e2e-test resourceVersion: "18947" uid: dae06e74-2185-4253-8cfa-8a4b08f5c551 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg namespace: kserve-ci-e2e-test resourceVersion: "18906" uid: 2d880044-04f1-4518-a423-305351e6122d kind: Event lastTimestamp: "2026-05-25T12:31:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:31:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:31:47Z" name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg.18b2ceea0584f987 namespace: kserve-ci-e2e-test resourceVersion: "18948" uid: fd6f79c0-c957-4272-ae70-6ba3349fc03d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg namespace: kserve-ci-e2e-test resourceVersion: "18906" uid: 2d880044-04f1-4518-a423-305351e6122d kind: Event lastTimestamp: "2026-05-25T12:31:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:31:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:31:47Z" name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg.18b2ceea05f4ef5e namespace: kserve-ci-e2e-test resourceVersion: "18949" uid: 94ebe26b-3c2b-41e6-ba7b-a3da90404f69 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:31:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg namespace: kserve-ci-e2e-test resourceVersion: "18906" uid: 2d880044-04f1-4518-a423-305351e6122d kind: Event lastTimestamp: "2026-05-25T12:32:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:31:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:24Z" name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg.18b2ceea69de49a3 namespace: kserve-ci-e2e-test resourceVersion: "19406" uid: d096e1e5-e82d-44fb-9734-d833f5c37435 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg namespace: kserve-ci-e2e-test resourceVersion: "18906" uid: 2d880044-04f1-4518-a423-305351e6122d kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:56Z" name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg.18b2cefa22b5a1eb namespace: kserve-ci-e2e-test resourceVersion: "19806" uid: d6c319f3-5057-48ad-93e0-1c83f72d26f4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg namespace: kserve-ci-e2e-test resourceVersion: "18906" uid: 2d880044-04f1-4518-a423-305351e6122d kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:56Z" name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg.18b2cefa22b72137 namespace: kserve-ci-e2e-test resourceVersion: "19810" uid: 8b48e8bf-0242-4da6-a98a-b7835092837c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg namespace: kserve-ci-e2e-test resourceVersion: "18906" uid: 2d880044-04f1-4518-a423-305351e6122d kind: Event lastTimestamp: "2026-05-25T12:32:59Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:32:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:59Z" name: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg.18b2cefaf25cada1 namespace: kserve-ci-e2e-test resourceVersion: "19883" uid: bdf01c28-7ccb-445a-9fe9-1c08f6feafd2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-bcff4-predictor-55cbf45559 namespace: kserve-ci-e2e-test resourceVersion: "18894" uid: b060caaa-e24e-4785-a6f2-1ae0e637d848 kind: Event lastTimestamp: "2026-05-25T12:31:46Z" message: 'Created pod: error-404-isvc-bcff4-predictor-55cbf45559-tw9hg' metadata: creationTimestamp: "2026-05-25T12:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:31:46Z" name: error-404-isvc-bcff4-predictor-55cbf45559.18b2cee9be1a5e52 namespace: kserve-ci-e2e-test resourceVersion: "18905" uid: 47064d31-1221-46e5-8de7-a6bbc1abecf3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-bcff4-predictor namespace: kserve-ci-e2e-test resourceVersion: "18889" uid: 78a69765-04b7-4936-a45c-c6609d5e5420 kind: Event lastTimestamp: "2026-05-25T12:31:46Z" message: Scaled up replica set error-404-isvc-bcff4-predictor-55cbf45559 from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:31:46Z" name: error-404-isvc-bcff4-predictor.18b2cee9bc1737ad namespace: kserve-ci-e2e-test resourceVersion: "18897" uid: 6cb0f29b-b506-466d-a40c-db43d32c78b4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bcff4-predictor namespace: kserve-ci-e2e-test resourceVersion: "18893" uid: 68c406eb-3ef0-41d4-b56a-9baa1c9725b5 kind: Event lastTimestamp: "2026-05-25T12:32:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:32:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:32:01Z" name: error-404-isvc-bcff4-predictor.18b2ceed3adfe566 namespace: kserve-ci-e2e-test resourceVersion: "19069" uid: af9eec4d-31bd-4465-9916-9b23aad0844a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bcff4-predictor namespace: kserve-ci-e2e-test resourceVersion: "18893" uid: 68c406eb-3ef0-41d4-b56a-9baa1c9725b5 kind: Event lastTimestamp: "2026-05-25T12:32:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:32:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:32:01Z" name: error-404-isvc-bcff4-predictor.18b2ceed3b5f94a8 namespace: kserve-ci-e2e-test resourceVersion: "19071" uid: 9eca085d-61a6-4a70-979c-70d4623c04aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:32:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bcff4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19070" uid: 68c406eb-3ef0-41d4-b56a-9baa1c9725b5 kind: Event lastTimestamp: "2026-05-25T12:32:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:32:46Z" name: error-404-isvc-bcff4-predictor.18b2cef0bc3f5798 namespace: kserve-ci-e2e-test resourceVersion: "19624" uid: 2cda0a73-b35d-48d6-9808-49dc37da873b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:32:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bcff4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19070" uid: 68c406eb-3ef0-41d4-b56a-9baa1c9725b5 kind: Event lastTimestamp: "2026-05-25T12:32:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:32:46Z" name: error-404-isvc-bcff4-predictor.18b2cef0bce0ae7c namespace: kserve-ci-e2e-test resourceVersion: "19626" uid: 23a1bf59-41a1-403f-9d0e-eb4f719b2bf7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:31:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bcff4 namespace: kserve-ci-e2e-test resourceVersion: "18861" uid: 3b59dbb1-a792-42ed-8152-92f7d426dcfb kind: Event lastTimestamp: "2026-05-25T12:32:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:32:35Z" name: error-404-isvc-bcff4.18b2cee9bc1f45ee namespace: kserve-ci-e2e-test resourceVersion: "19507" uid: b8cfbaf6-c6a7-4135-902f-822dcd344af3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bcff4 namespace: kserve-ci-e2e-test resourceVersion: "18861" uid: 3b59dbb1-a792-42ed-8152-92f7d426dcfb kind: Event lastTimestamp: "2026-05-25T12:31:46Z" message: 'Failed to update status for InferenceService "error-404-isvc-bcff4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bcff4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:31:46Z" name: error-404-isvc-bcff4.18b2cee9bcba2a4a namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: 7f6aed68-a0f1-4303-8272-dd15d1cd33b4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bcff4 namespace: kserve-ci-e2e-test resourceVersion: "18861" uid: 3b59dbb1-a792-42ed-8152-92f7d426dcfb kind: Event lastTimestamp: "2026-05-25T12:31:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bcff4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:31:46Z" name: error-404-isvc-bcff4.18b2cee9bcba82b9 namespace: kserve-ci-e2e-test resourceVersion: "18902" uid: a5e03e55-fbb1-4fe1-b5b9-b608124198a2 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bcff4 namespace: kserve-ci-e2e-test resourceVersion: "19505" uid: 3b59dbb1-a792-42ed-8152-92f7d426dcfb kind: Event lastTimestamp: "2026-05-25T12:32:35Z" message: InferenceService [error-404-isvc-bcff4] is Ready metadata: creationTimestamp: "2026-05-25T12:32:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:32:35Z" name: error-404-isvc-bcff4.18b2cef526b5ac3f namespace: kserve-ci-e2e-test resourceVersion: "19506" uid: a118d119-84d9-42b5-9801-34d58311a06a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:42:21.335852Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh namespace: kserve-ci-e2e-test resourceVersion: "24243" uid: 4ab201cd-f34e-4d4c-aa98-179dee7a9ede kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh to ip-10-0-142-53.ec2.internal metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:42:21Z" name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh.18b2cf7da67d4829 namespace: kserve-ci-e2e-test resourceVersion: "24247" uid: 53fe736b-f6a3-42ec-a1a0-9ff77bdd4f86 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh namespace: kserve-ci-e2e-test resourceVersion: "24250" uid: 4ab201cd-f34e-4d4c-aa98-179dee7a9ede kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: Add eth0 [10.134.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:42:21Z" name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh.18b2cf7dc0f7f7c9 namespace: kserve-ci-e2e-test resourceVersion: "24265" uid: 872164ef-ffa0-473f-a615-efc718174186 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh namespace: kserve-ci-e2e-test resourceVersion: "24245" uid: 4ab201cd-f34e-4d4c-aa98-179dee7a9ede kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:21Z" name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh.18b2cf7dc27621ec namespace: kserve-ci-e2e-test resourceVersion: "24267" uid: 4956cb66-d212-45c5-bb1b-7b26d9acc40c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh namespace: kserve-ci-e2e-test resourceVersion: "24245" uid: 4ab201cd-f34e-4d4c-aa98-179dee7a9ede kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:21Z" name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh.18b2cf7dc6893838 namespace: kserve-ci-e2e-test resourceVersion: "24269" uid: 297901f8-4ff8-4680-9172-a29910403409 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh namespace: kserve-ci-e2e-test resourceVersion: "24245" uid: 4ab201cd-f34e-4d4c-aa98-179dee7a9ede kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:21Z" name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh.18b2cf7dc6fa7049 namespace: kserve-ci-e2e-test resourceVersion: "24270" uid: 648e1d63-8a6d-4a87-897b-a9c8b534a7f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh namespace: kserve-ci-e2e-test resourceVersion: "24245" uid: 4ab201cd-f34e-4d4c-aa98-179dee7a9ede kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:21Z" name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh.18b2cf7dc707970e namespace: kserve-ci-e2e-test resourceVersion: "24272" uid: 0836acb6-c719-4599-afdf-2286d3c27123 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh namespace: kserve-ci-e2e-test resourceVersion: "24245" uid: 4ab201cd-f34e-4d4c-aa98-179dee7a9ede kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:21Z" name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh.18b2cf7dca201620 namespace: kserve-ci-e2e-test resourceVersion: "24274" uid: 802dac00-b027-437d-9cff-dffb3aecf833 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh namespace: kserve-ci-e2e-test resourceVersion: "24245" uid: 4ab201cd-f34e-4d4c-aa98-179dee7a9ede kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:21Z" name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh.18b2cf7dca9bf656 namespace: kserve-ci-e2e-test resourceVersion: "24275" uid: 52c484a7-87dc-47c1-b0ac-13d8da7ce91b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:42:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh namespace: kserve-ci-e2e-test resourceVersion: "24245" uid: 4ab201cd-f34e-4d4c-aa98-179dee7a9ede kind: Event lastTimestamp: "2026-05-25T12:42:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.49:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:42:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:59Z" name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh.18b2cf7e20c5ae52 namespace: kserve-ci-e2e-test resourceVersion: "24727" uid: acbefd6b-2484-4ca7-ab56-9ce0d1cf7cf4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh namespace: kserve-ci-e2e-test resourceVersion: "24245" uid: 4ab201cd-f34e-4d4c-aa98-179dee7a9ede kind: Event lastTimestamp: "2026-05-25T12:43:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:31Z" name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh.18b2cf8dfd0a830b namespace: kserve-ci-e2e-test resourceVersion: "25086" uid: 283c48a0-96e9-4ba1-b5d5-721d2a6b4c0c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh namespace: kserve-ci-e2e-test resourceVersion: "24245" uid: 4ab201cd-f34e-4d4c-aa98-179dee7a9ede kind: Event lastTimestamp: "2026-05-25T12:43:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:31Z" name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh.18b2cf8dfd0b176b namespace: kserve-ci-e2e-test resourceVersion: "25087" uid: 276cdc91-bad8-45e3-8c52-a2d138aebaae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh namespace: kserve-ci-e2e-test resourceVersion: "24245" uid: 4ab201cd-f34e-4d4c-aa98-179dee7a9ede kind: Event lastTimestamp: "2026-05-25T12:43:34Z" message: 'Readiness probe failed: Get "https://10.134.0.49:8643/healthz": dial tcp 10.134.0.49:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:34Z" name: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh.18b2cf8ea8f64863 namespace: kserve-ci-e2e-test resourceVersion: "25206" uid: 9d082fee-180f-47bd-b072-0f0db02570a5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-fd4b4-predictor-7694f9fcbf namespace: kserve-ci-e2e-test resourceVersion: "24235" uid: aa4c4dcc-3cc6-4d4c-a541-d9f95430fcaf kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: 'Created pod: error-404-isvc-fd4b4-predictor-7694f9fcbf-n88zh' metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:42:21Z" name: error-404-isvc-fd4b4-predictor-7694f9fcbf.18b2cf7da6135f2c namespace: kserve-ci-e2e-test resourceVersion: "24244" uid: b99eddc3-8224-4516-80cc-458217e13374 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-fd4b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "24232" uid: 42678b39-9253-43d4-abf4-c393ac3ac297 kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: Scaled up replica set error-404-isvc-fd4b4-predictor-7694f9fcbf from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:42:21Z" name: error-404-isvc-fd4b4-predictor.18b2cf7da478b32e namespace: kserve-ci-e2e-test resourceVersion: "24237" uid: 875e107d-5fd8-4c90-bf6a-37467e113fd0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:42:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fd4b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "24236" uid: f9f813d3-9d10-490c-971d-d27f78bfc8ec kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:42:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:42:51Z" name: error-404-isvc-fd4b4-predictor.18b2cf812354b18b namespace: kserve-ci-e2e-test resourceVersion: "24623" uid: 93b06d51-1a71-4a37-bf10-46c3498f38f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:42:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fd4b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "24236" uid: f9f813d3-9d10-490c-971d-d27f78bfc8ec kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:42:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:42:51Z" name: error-404-isvc-fd4b4-predictor.18b2cf8123c7b416 namespace: kserve-ci-e2e-test resourceVersion: "24626" uid: 0fcd4f99-3e88-413e-808e-0ab837716051 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:43:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fd4b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "24412" uid: f9f813d3-9d10-490c-971d-d27f78bfc8ec kind: Event lastTimestamp: "2026-05-25T12:43:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:43:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:43:21Z" name: error-404-isvc-fd4b4-predictor.18b2cf8821434eac namespace: kserve-ci-e2e-test resourceVersion: "24952" uid: 51aa0e7a-112a-458f-b8ae-f26875ddb9b6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:43:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fd4b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "24412" uid: f9f813d3-9d10-490c-971d-d27f78bfc8ec kind: Event lastTimestamp: "2026-05-25T12:43:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:43:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:43:21Z" name: error-404-isvc-fd4b4-predictor.18b2cf8821b5a8b9 namespace: kserve-ci-e2e-test resourceVersion: "24954" uid: d396c3e5-888d-4050-8bb1-370696154628 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fd4b4 namespace: kserve-ci-e2e-test resourceVersion: "24203" uid: 2d0d263c-1339-4029-ad60-b89200419aa9 kind: Event lastTimestamp: "2026-05-25T12:43:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:43:09Z" name: error-404-isvc-fd4b4.18b2cf7da4aa0e89 namespace: kserve-ci-e2e-test resourceVersion: "24838" uid: 2f891cd6-f77e-4574-b185-b919fe973a6b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fd4b4 namespace: kserve-ci-e2e-test resourceVersion: "24203" uid: 2d0d263c-1339-4029-ad60-b89200419aa9 kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: 'Failed to update status for InferenceService "error-404-isvc-fd4b4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fd4b4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:42:21Z" name: error-404-isvc-fd4b4.18b2cf7da521bb10 namespace: kserve-ci-e2e-test resourceVersion: "24240" uid: f11d25fa-11e6-4a98-9bde-94d345d69e4e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fd4b4 namespace: kserve-ci-e2e-test resourceVersion: "24203" uid: 2d0d263c-1339-4029-ad60-b89200419aa9 kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fd4b4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:42:21Z" name: error-404-isvc-fd4b4.18b2cf7da5220f87 namespace: kserve-ci-e2e-test resourceVersion: "24242" uid: 7518be5a-18b8-443b-b967-1bf1c1072e86 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fd4b4 namespace: kserve-ci-e2e-test resourceVersion: "24830" uid: 2d0d263c-1339-4029-ad60-b89200419aa9 kind: Event lastTimestamp: "2026-05-25T12:43:09Z" message: InferenceService [error-404-isvc-fd4b4] is Ready metadata: creationTimestamp: "2026-05-25T12:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:43:09Z" name: error-404-isvc-fd4b4.18b2cf88dc9c0ed1 namespace: kserve-ci-e2e-test resourceVersion: "24831" uid: b828d5f7-4fed-44b6-8125-afc5a8d66592 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:21:00.880999Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb namespace: kserve-ci-e2e-test resourceVersion: "13385" uid: 2640504a-5b87-49a4-97b7-0e46974ca1ac kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-678784f46b-b29kb to ip-10-0-133-4.ec2.internal metadata: creationTimestamp: "2026-05-25T12:21:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:21:00Z" name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb.18b2ce53856ec771 namespace: kserve-ci-e2e-test resourceVersion: "13390" uid: b7a46a79-390a-40fa-bb17-5368d4c656f6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:21:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb namespace: kserve-ci-e2e-test resourceVersion: "13387" uid: 2640504a-5b87-49a4-97b7-0e46974ca1ac kind: Event lastTimestamp: "2026-05-25T12:21:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:21:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:01Z" name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb.18b2ce539145ed57 namespace: kserve-ci-e2e-test resourceVersion: "13490" uid: c58bed00-7d93-451b-9392-f4839c9b78cc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb namespace: kserve-ci-e2e-test resourceVersion: "13396" uid: 2640504a-5b87-49a4-97b7-0e46974ca1ac kind: Event lastTimestamp: "2026-05-25T12:21:02Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:21:02Z" name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb.18b2ce53f7d1330a namespace: kserve-ci-e2e-test resourceVersion: "13534" uid: ed063938-08ca-490f-8cef-e85821c247e0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb namespace: kserve-ci-e2e-test resourceVersion: "13387" uid: 2640504a-5b87-49a4-97b7-0e46974ca1ac kind: Event lastTimestamp: "2026-05-25T12:21:02Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:4e534c6f6e9d80dfdaea245073aaa55f0290d59a851159e578fe713a363df99a" metadata: creationTimestamp: "2026-05-25T12:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:02Z" name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb.18b2ce53f9f92d97 namespace: kserve-ci-e2e-test resourceVersion: "13537" uid: b8685c2f-8512-4f07-a877-fc9a5c5c9c64 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb namespace: kserve-ci-e2e-test resourceVersion: "13387" uid: 2640504a-5b87-49a4-97b7-0e46974ca1ac kind: Event lastTimestamp: "2026-05-25T12:21:06Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:4e534c6f6e9d80dfdaea245073aaa55f0290d59a851159e578fe713a363df99a" in 3.223s (3.223s including waiting). Image size: 301472374 bytes.' metadata: creationTimestamp: "2026-05-25T12:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:06Z" name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb.18b2ce54ba163e74 namespace: kserve-ci-e2e-test resourceVersion: "13576" uid: 449d5842-2a23-43e6-a0c6-fa828a6f308e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb namespace: kserve-ci-e2e-test resourceVersion: "13387" uid: 2640504a-5b87-49a4-97b7-0e46974ca1ac kind: Event lastTimestamp: "2026-05-25T12:21:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-25T12:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:06Z" name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb.18b2ce54bd6367ab namespace: kserve-ci-e2e-test resourceVersion: "13580" uid: 8bba9524-3888-4307-b9fa-671ea908126f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb namespace: kserve-ci-e2e-test resourceVersion: "13387" uid: 2640504a-5b87-49a4-97b7-0e46974ca1ac kind: Event lastTimestamp: "2026-05-25T12:21:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-25T12:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:06Z" name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb.18b2ce54bdfbd454 namespace: kserve-ci-e2e-test resourceVersion: "13583" uid: 3fd60c77-e028-4a67-ae33-64fb29dee9ca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb namespace: kserve-ci-e2e-test resourceVersion: "13387" uid: 2640504a-5b87-49a4-97b7-0e46974ca1ac kind: Event lastTimestamp: "2026-05-25T12:21:09Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" metadata: creationTimestamp: "2026-05-25T12:21:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:09Z" name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb.18b2ce559f8cef2e namespace: kserve-ci-e2e-test resourceVersion: "13629" uid: 2762ce77-1506-4150-a05d-b5a4f3076535 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb namespace: kserve-ci-e2e-test resourceVersion: "13387" uid: 2640504a-5b87-49a4-97b7-0e46974ca1ac kind: Event lastTimestamp: "2026-05-25T12:21:23Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" in 13.932s (13.932s including waiting). Image size: 1560933302 bytes.' metadata: creationTimestamp: "2026-05-25T12:21:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:23Z" name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb.18b2ce58ddfc77e3 namespace: kserve-ci-e2e-test resourceVersion: "13782" uid: a801a5b3-9ad0-41f6-b086-2b13d0caf3c1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb namespace: kserve-ci-e2e-test resourceVersion: "13387" uid: 2640504a-5b87-49a4-97b7-0e46974ca1ac kind: Event lastTimestamp: "2026-05-25T12:21:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:21:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:23Z" name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb.18b2ce58e25c53b0 namespace: kserve-ci-e2e-test resourceVersion: "13784" uid: 642b53b3-caa7-45dd-9686-3ccf8fd5ff5f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb namespace: kserve-ci-e2e-test resourceVersion: "13387" uid: 2640504a-5b87-49a4-97b7-0e46974ca1ac kind: Event lastTimestamp: "2026-05-25T12:21:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:21:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:23Z" name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb.18b2ce58e2c0b841 namespace: kserve-ci-e2e-test resourceVersion: "13787" uid: bd44f444-6947-4e7e-b3fc-5b2787131686 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb namespace: kserve-ci-e2e-test resourceVersion: "13387" uid: 2640504a-5b87-49a4-97b7-0e46974ca1ac kind: Event lastTimestamp: "2026-05-25T12:21:23Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-25T12:21:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:23Z" name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb.18b2ce58e2cdd2f3 namespace: kserve-ci-e2e-test resourceVersion: "13788" uid: 486490b4-855e-4e54-8887-0da838e7e0ca reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb namespace: kserve-ci-e2e-test resourceVersion: "13387" uid: 2640504a-5b87-49a4-97b7-0e46974ca1ac kind: Event lastTimestamp: "2026-05-25T12:21:26Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.403s (2.403s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-25T12:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:26Z" name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb.18b2ce5972095532 namespace: kserve-ci-e2e-test resourceVersion: "13806" uid: cda487e7-a051-45c7-9c79-ea4229a9dbb9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb namespace: kserve-ci-e2e-test resourceVersion: "13387" uid: 2640504a-5b87-49a4-97b7-0e46974ca1ac kind: Event lastTimestamp: "2026-05-25T12:21:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:26Z" name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb.18b2ce59756f7ce5 namespace: kserve-ci-e2e-test resourceVersion: "13809" uid: 76f8d91a-e1be-4e89-91b0-83acf4e42f17 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb namespace: kserve-ci-e2e-test resourceVersion: "13387" uid: 2640504a-5b87-49a4-97b7-0e46974ca1ac kind: Event lastTimestamp: "2026-05-25T12:21:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:26Z" name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb.18b2ce59761d4c36 namespace: kserve-ci-e2e-test resourceVersion: "13812" uid: eeb5dd1b-8219-4b72-929a-71809b4d15c6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-25T12:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb namespace: kserve-ci-e2e-test resourceVersion: "13387" uid: 2640504a-5b87-49a4-97b7-0e46974ca1ac kind: Event lastTimestamp: "2026-05-25T12:23:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:03Z" name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb.18b2ce59d4868b5d namespace: kserve-ci-e2e-test resourceVersion: "15056" uid: aec57f39-287f-432c-b685-5f4023726143 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb namespace: kserve-ci-e2e-test resourceVersion: "13387" uid: 2640504a-5b87-49a4-97b7-0e46974ca1ac kind: Event lastTimestamp: "2026-05-25T12:23:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:01Z" name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb.18b2ce6f821fc17f namespace: kserve-ci-e2e-test resourceVersion: "14962" uid: e99f3f75-69e4-46a7-a8d8-134bf80333a2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb namespace: kserve-ci-e2e-test resourceVersion: "13387" uid: 2640504a-5b87-49a4-97b7-0e46974ca1ac kind: Event lastTimestamp: "2026-05-25T12:23:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:01Z" name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb.18b2ce6f822077e1 namespace: kserve-ci-e2e-test resourceVersion: "14965" uid: d9b29602-cbaa-4f97-b2e2-7699f41f9d38 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb namespace: kserve-ci-e2e-test resourceVersion: "13387" uid: 2640504a-5b87-49a4-97b7-0e46974ca1ac kind: Event lastTimestamp: "2026-05-25T12:23:03Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:03Z" name: isvc-sklearn-graph-1-predictor-678784f46b-b29kb.18b2ce702eb5dab7 namespace: kserve-ci-e2e-test resourceVersion: "15055" uid: 34a3ca88-e036-43b3-a5e0-a28d124fdb6d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-678784f46b namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: f625f569-adee-4b48-9395-edccf4bba63b kind: Event lastTimestamp: "2026-05-25T12:21:00Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-678784f46b-b29kb' metadata: creationTimestamp: "2026-05-25T12:21:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:21:00Z" name: isvc-sklearn-graph-1-predictor-678784f46b.18b2ce5384fa363b namespace: kserve-ci-e2e-test resourceVersion: "13386" uid: b42e3f64-380e-4364-bf83-e5261be5000a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13372" uid: 74ac3df8-2b57-4b65-8868-fa97915fd544 kind: Event lastTimestamp: "2026-05-25T12:21:00Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-678784f46b from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:21:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:21:00Z" name: isvc-sklearn-graph-1-predictor.18b2ce537d6079f7 namespace: kserve-ci-e2e-test resourceVersion: "13375" uid: 6e136686-fb7e-402f-91bd-d93dd9044d37 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T12:21:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13380" uid: b6ab4701-8371-438a-ac2e-a591bb833e61 kind: Event lastTimestamp: "2026-05-25T12:22:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:00Z" name: isvc-sklearn-graph-1-predictor.18b2ce570346faa0 namespace: kserve-ci-e2e-test resourceVersion: "14111" uid: 1ff198ca-4dfc-4d18-98c9-22647dda496e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T12:21:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13380" uid: b6ab4701-8371-438a-ac2e-a591bb833e61 kind: Event lastTimestamp: "2026-05-25T12:22:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:00Z" name: isvc-sklearn-graph-1-predictor.18b2ce5703bbf03b namespace: kserve-ci-e2e-test resourceVersion: "14112" uid: c84d3715-f517-43b8-b6c7-3a9b20a35d8c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T12:22:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13682" uid: b6ab4701-8371-438a-ac2e-a591bb833e61 kind: Event lastTimestamp: "2026-05-25T12:23:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:22:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:23:01Z" name: isvc-sklearn-graph-1-predictor.18b2ce64fffeb3da namespace: kserve-ci-e2e-test resourceVersion: "14926" uid: 48834f08-5819-4270-bfdd-0117e16812ae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T12:22:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13682" uid: b6ab4701-8371-438a-ac2e-a591bb833e61 kind: Event lastTimestamp: "2026-05-25T12:23:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:22:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:23:01Z" name: isvc-sklearn-graph-1-predictor.18b2ce650073c49c namespace: kserve-ci-e2e-test resourceVersion: "14931" uid: 76bb0ff9-c346-44b9-902f-3519adf77795 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-25T12:21:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13356" uid: fb78c3c7-d623-4720-89b0-80ab6b3e385f kind: Event lastTimestamp: "2026-05-25T12:22:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:21:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:22:24Z" name: isvc-sklearn-graph-1.18b2ce53839c79f4 namespace: kserve-ci-e2e-test resourceVersion: "14359" uid: c23022f8-7743-437d-9322-d15b941b338b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14357" uid: fb78c3c7-d623-4720-89b0-80ab6b3e385f kind: Event lastTimestamp: "2026-05-25T12:22:24Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-05-25T12:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:22:24Z" name: isvc-sklearn-graph-1.18b2ce66e418d0ef namespace: kserve-ci-e2e-test resourceVersion: "14358" uid: 898fa38e-29d9-4e51-8507-c99111d1c87a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:21:01.939644Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj namespace: kserve-ci-e2e-test resourceVersion: "13501" uid: bb8be00d-f8e0-4263-afc5-8d62a7ecd0e4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj to ip-10-0-133-4.ec2.internal metadata: creationTimestamp: "2026-05-25T12:21:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:21:01Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj.18b2ce53c4887181 namespace: kserve-ci-e2e-test resourceVersion: "13505" uid: 03de801b-d79d-4a60-a039-2357bae4dde4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj namespace: kserve-ci-e2e-test resourceVersion: "13512" uid: bb8be00d-f8e0-4263-afc5-8d62a7ecd0e4 kind: Event lastTimestamp: "2026-05-25T12:21:02Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:21:02Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj.18b2ce53dd674abc namespace: kserve-ci-e2e-test resourceVersion: "13520" uid: c1f61087-19af-47e8-abf5-12a1df26f447 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj namespace: kserve-ci-e2e-test resourceVersion: "13502" uid: bb8be00d-f8e0-4263-afc5-8d62a7ecd0e4 kind: Event lastTimestamp: "2026-05-25T12:21:02Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:4e534c6f6e9d80dfdaea245073aaa55f0290d59a851159e578fe713a363df99a" metadata: creationTimestamp: "2026-05-25T12:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:02Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj.18b2ce53dfb1d2c7 namespace: kserve-ci-e2e-test resourceVersion: "13525" uid: a16d86a0-af26-4150-8ffc-03e8bac9c42d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj namespace: kserve-ci-e2e-test resourceVersion: "13502" uid: bb8be00d-f8e0-4263-afc5-8d62a7ecd0e4 kind: Event lastTimestamp: "2026-05-25T12:21:06Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:4e534c6f6e9d80dfdaea245073aaa55f0290d59a851159e578fe713a363df99a" in 3.673s (3.673s including waiting). Image size: 301472374 bytes.' metadata: creationTimestamp: "2026-05-25T12:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:06Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj.18b2ce54baae76f6 namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: 20096b65-4727-47fa-bc10-8bb4d1a72c75 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj namespace: kserve-ci-e2e-test resourceVersion: "13502" uid: bb8be00d-f8e0-4263-afc5-8d62a7ecd0e4 kind: Event lastTimestamp: "2026-05-25T12:21:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-25T12:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:06Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj.18b2ce54bd6f7ef6 namespace: kserve-ci-e2e-test resourceVersion: "13581" uid: 6852b592-ecaa-4a04-b29f-a8fadd97d9c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj namespace: kserve-ci-e2e-test resourceVersion: "13502" uid: bb8be00d-f8e0-4263-afc5-8d62a7ecd0e4 kind: Event lastTimestamp: "2026-05-25T12:21:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-25T12:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:06Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj.18b2ce54bddf8069 namespace: kserve-ci-e2e-test resourceVersion: "13582" uid: f0103ea6-d377-42b7-a24b-ddf7ab4336d9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj namespace: kserve-ci-e2e-test resourceVersion: "13502" uid: bb8be00d-f8e0-4263-afc5-8d62a7ecd0e4 kind: Event lastTimestamp: "2026-05-25T12:21:09Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" metadata: creationTimestamp: "2026-05-25T12:21:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:09Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj.18b2ce559f9f4719 namespace: kserve-ci-e2e-test resourceVersion: "13630" uid: 595415cb-de00-41a4-a29a-d64ad0e78e00 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj namespace: kserve-ci-e2e-test resourceVersion: "13502" uid: bb8be00d-f8e0-4263-afc5-8d62a7ecd0e4 kind: Event lastTimestamp: "2026-05-25T12:21:23Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" in 13.935s (13.935s including waiting). Image size: 1560933302 bytes.' metadata: creationTimestamp: "2026-05-25T12:21:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:23Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj.18b2ce58de42333d namespace: kserve-ci-e2e-test resourceVersion: "13783" uid: 820e83f5-e69c-4631-b14c-ed1c30d5ce0d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj namespace: kserve-ci-e2e-test resourceVersion: "13502" uid: bb8be00d-f8e0-4263-afc5-8d62a7ecd0e4 kind: Event lastTimestamp: "2026-05-25T12:21:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:21:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:23Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj.18b2ce58e25f5129 namespace: kserve-ci-e2e-test resourceVersion: "13785" uid: e011623c-192d-4eec-94a3-2187fe5fdfb4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj namespace: kserve-ci-e2e-test resourceVersion: "13502" uid: bb8be00d-f8e0-4263-afc5-8d62a7ecd0e4 kind: Event lastTimestamp: "2026-05-25T12:21:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:21:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:23Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj.18b2ce58e2c0bde5 namespace: kserve-ci-e2e-test resourceVersion: "13786" uid: aa995c2a-9dc0-47cc-bc79-885898bd3b8a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj namespace: kserve-ci-e2e-test resourceVersion: "13502" uid: bb8be00d-f8e0-4263-afc5-8d62a7ecd0e4 kind: Event lastTimestamp: "2026-05-25T12:21:23Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-25T12:21:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:23Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj.18b2ce58e2ceb6a4 namespace: kserve-ci-e2e-test resourceVersion: "13789" uid: fb285083-292c-4549-ae0e-53286fb71141 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj namespace: kserve-ci-e2e-test resourceVersion: "13502" uid: bb8be00d-f8e0-4263-afc5-8d62a7ecd0e4 kind: Event lastTimestamp: "2026-05-25T12:21:26Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.405s (2.405s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-25T12:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:26Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj.18b2ce59722b20ff namespace: kserve-ci-e2e-test resourceVersion: "13807" uid: b37f49b0-cb15-4bb2-9830-ae9a0add16fc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj namespace: kserve-ci-e2e-test resourceVersion: "13502" uid: bb8be00d-f8e0-4263-afc5-8d62a7ecd0e4 kind: Event lastTimestamp: "2026-05-25T12:21:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:26Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj.18b2ce59757d541a namespace: kserve-ci-e2e-test resourceVersion: "13810" uid: 3df92378-be1a-4624-bfcd-97464ff46ef6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj namespace: kserve-ci-e2e-test resourceVersion: "13502" uid: bb8be00d-f8e0-4263-afc5-8d62a7ecd0e4 kind: Event lastTimestamp: "2026-05-25T12:21:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:26Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj.18b2ce5975de531e namespace: kserve-ci-e2e-test resourceVersion: "13811" uid: 7a2fb498-4c59-4474-818f-6f7a37cc1b2e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-25T12:21:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj namespace: kserve-ci-e2e-test resourceVersion: "13502" uid: bb8be00d-f8e0-4263-afc5-8d62a7ecd0e4 kind: Event lastTimestamp: "2026-05-25T12:23:02Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:21:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:02Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj.18b2ce5998c0451d namespace: kserve-ci-e2e-test resourceVersion: "15040" uid: a8497088-1351-4571-a385-87a24dc907b5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj namespace: kserve-ci-e2e-test resourceVersion: "13502" uid: bb8be00d-f8e0-4263-afc5-8d62a7ecd0e4 kind: Event lastTimestamp: "2026-05-25T12:23:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:01Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj.18b2ce6f7d5e9902 namespace: kserve-ci-e2e-test resourceVersion: "14920" uid: e65dfca0-dac2-458d-93af-acb134036123 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj namespace: kserve-ci-e2e-test resourceVersion: "13502" uid: bb8be00d-f8e0-4263-afc5-8d62a7ecd0e4 kind: Event lastTimestamp: "2026-05-25T12:23:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:01Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj.18b2ce6f7d5f6950 namespace: kserve-ci-e2e-test resourceVersion: "14923" uid: 103c060a-0d63-48c1-95e5-288d08d0d75e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj namespace: kserve-ci-e2e-test resourceVersion: "13502" uid: bb8be00d-f8e0-4263-afc5-8d62a7ecd0e4 kind: Event lastTimestamp: "2026-05-25T12:23:02Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:02Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj.18b2ce6ff2f491bd namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: d04464f2-69e1-43cd-820a-0f534d22769c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-4.ec2.internal source: component: kubelet host: ip-10-0-133-4.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-54c99bfc7d namespace: kserve-ci-e2e-test resourceVersion: "13484" uid: 7883da70-257b-4177-995d-58241608c58e kind: Event lastTimestamp: "2026-05-25T12:21:01Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-54c99bfc7d-v6cdj' metadata: creationTimestamp: "2026-05-25T12:21:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:21:01Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d.18b2ce53c412fa35 namespace: kserve-ci-e2e-test resourceVersion: "13503" uid: d45f8c53-906c-4fb7-8c0b-da085399e679 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13483" uid: dc9bae9e-7640-42a8-8e25-edbb8cb09973 kind: Event lastTimestamp: "2026-05-25T12:21:01Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-54c99bfc7d from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:21:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:21:01Z" name: isvc-sklearn-graph-2-predictor.18b2ce53aec305bd namespace: kserve-ci-e2e-test resourceVersion: "13486" uid: 90c8e3ed-1842-417b-9f4c-e63ba8219fe4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:21:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13485" uid: 33cd189b-43b0-43a5-9e26-76a79de6afbb kind: Event lastTimestamp: "2026-05-25T12:21:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:21:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:21:46Z" name: isvc-sklearn-graph-2-predictor.18b2ce572e2190e1 namespace: kserve-ci-e2e-test resourceVersion: "14002" uid: 643cf812-acde-4441-8e51-823f7525ae0d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:21:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13485" uid: 33cd189b-43b0-43a5-9e26-76a79de6afbb kind: Event lastTimestamp: "2026-05-25T12:21:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:21:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:21:46Z" name: isvc-sklearn-graph-2-predictor.18b2ce572eb38f18 namespace: kserve-ci-e2e-test resourceVersion: "14003" uid: 2966c1bb-e26e-49c5-97a4-d814b116f1e3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T12:22:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13701" uid: 33cd189b-43b0-43a5-9e26-76a79de6afbb kind: Event lastTimestamp: "2026-05-25T12:22:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:46Z" name: isvc-sklearn-graph-2-predictor.18b2ce61ab1e4ea7 namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: 31ffbd9a-5248-410d-9110-fc238c11cfa0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T12:22:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13701" uid: 33cd189b-43b0-43a5-9e26-76a79de6afbb kind: Event lastTimestamp: "2026-05-25T12:22:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:46Z" name: isvc-sklearn-graph-2-predictor.18b2ce61abb8f504 namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: 012241c5-d1a2-4467-80ae-479fbe785b07 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-25T12:21:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13463" uid: 58cb04e9-7dbd-4154-bdea-aa661c54b8ba kind: Event lastTimestamp: "2026-05-25T12:22:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:21:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:22:33Z" name: isvc-sklearn-graph-2.18b2ce53af0cc2fe namespace: kserve-ci-e2e-test resourceVersion: "14592" uid: d8eb40f8-0675-4e9b-8524-f30f1175c323 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: 58cb04e9-7dbd-4154-bdea-aa661c54b8ba kind: Event lastTimestamp: "2026-05-25T12:22:33Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-05-25T12:22:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:22:33Z" name: isvc-sklearn-graph-2.18b2ce68fc531a17 namespace: kserve-ci-e2e-test resourceVersion: "14591" uid: b4e61f46-cfbe-445f-bb41-77c89bc899ec reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:21:01.344993Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5 namespace: kserve-ci-e2e-test resourceVersion: "13466" uid: 35d8edba-6dc8-441f-b65f-9d4e667b4dcf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-x8tg5 to ip-10-0-143-107.ec2.internal metadata: creationTimestamp: "2026-05-25T12:21:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:21:01Z" name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5.18b2ce53a1174797 namespace: kserve-ci-e2e-test resourceVersion: "13470" uid: 89f22570-b393-48b4-b91a-67a99ad0d620 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5 namespace: kserve-ci-e2e-test resourceVersion: "13468" uid: 35d8edba-6dc8-441f-b65f-9d4e667b4dcf kind: Event lastTimestamp: "2026-05-25T12:21:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:21:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:01Z" name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5.18b2ce53acd16482 namespace: kserve-ci-e2e-test resourceVersion: "13479" uid: 0e5ab3cc-60ef-4914-aa20-6c76df9cdcb6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5 namespace: kserve-ci-e2e-test resourceVersion: "13474" uid: 35d8edba-6dc8-441f-b65f-9d4e667b4dcf kind: Event lastTimestamp: "2026-05-25T12:21:02Z" message: Add eth0 [10.132.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:21:02Z" name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5.18b2ce53de68c996 namespace: kserve-ci-e2e-test resourceVersion: "13523" uid: 0fe076b6-f5bd-413b-9194-c8e3e4fefaa5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5 namespace: kserve-ci-e2e-test resourceVersion: "13468" uid: 35d8edba-6dc8-441f-b65f-9d4e667b4dcf kind: Event lastTimestamp: "2026-05-25T12:21:02Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:4e534c6f6e9d80dfdaea245073aaa55f0290d59a851159e578fe713a363df99a" metadata: creationTimestamp: "2026-05-25T12:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:02Z" name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5.18b2ce53e0f552db namespace: kserve-ci-e2e-test resourceVersion: "13527" uid: bcd03086-6bcc-4658-a809-40a0d23dc7f0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5 namespace: kserve-ci-e2e-test resourceVersion: "13468" uid: 35d8edba-6dc8-441f-b65f-9d4e667b4dcf kind: Event lastTimestamp: "2026-05-25T12:21:06Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:4e534c6f6e9d80dfdaea245073aaa55f0290d59a851159e578fe713a363df99a" in 3.675s (3.675s including waiting). Image size: 301472374 bytes.' metadata: creationTimestamp: "2026-05-25T12:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:06Z" name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5.18b2ce54bc06ff26 namespace: kserve-ci-e2e-test resourceVersion: "13579" uid: d8c5deee-9797-49e5-8483-ab7d1a9a76de reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5 namespace: kserve-ci-e2e-test resourceVersion: "13468" uid: 35d8edba-6dc8-441f-b65f-9d4e667b4dcf kind: Event lastTimestamp: "2026-05-25T12:21:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-25T12:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:06Z" name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5.18b2ce54bf3f5c93 namespace: kserve-ci-e2e-test resourceVersion: "13584" uid: 5737b984-8aa2-49aa-95c2-01e772fbc87b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5 namespace: kserve-ci-e2e-test resourceVersion: "13468" uid: 35d8edba-6dc8-441f-b65f-9d4e667b4dcf kind: Event lastTimestamp: "2026-05-25T12:21:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-25T12:21:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:06Z" name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5.18b2ce54bfb19f8f namespace: kserve-ci-e2e-test resourceVersion: "13585" uid: 5b5fac7d-aae8-4bcc-b281-71edfbb8c8fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5 namespace: kserve-ci-e2e-test resourceVersion: "13468" uid: 35d8edba-6dc8-441f-b65f-9d4e667b4dcf kind: Event lastTimestamp: "2026-05-25T12:21:10Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-05-25T12:21:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:10Z" name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5.18b2ce55be311eff namespace: kserve-ci-e2e-test resourceVersion: "13644" uid: 21c06bbc-06c7-4b6a-ae37-06cd0c67f9c6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5 namespace: kserve-ci-e2e-test resourceVersion: "13468" uid: 35d8edba-6dc8-441f-b65f-9d4e667b4dcf kind: Event lastTimestamp: "2026-05-25T12:21:27Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 17.283s (17.283s including waiting). Image size: 1306319098 bytes.' metadata: creationTimestamp: "2026-05-25T12:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5.18b2ce59c4648569 namespace: kserve-ci-e2e-test resourceVersion: "13830" uid: 08ba5479-57a0-4150-84cc-a6fc6f5dc052 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5 namespace: kserve-ci-e2e-test resourceVersion: "13468" uid: 35d8edba-6dc8-441f-b65f-9d4e667b4dcf kind: Event lastTimestamp: "2026-05-25T12:21:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5.18b2ce59c74bc034 namespace: kserve-ci-e2e-test resourceVersion: "13832" uid: cc34b3a0-bb90-499a-9643-f67e39575f3d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5 namespace: kserve-ci-e2e-test resourceVersion: "13468" uid: 35d8edba-6dc8-441f-b65f-9d4e667b4dcf kind: Event lastTimestamp: "2026-05-25T12:21:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5.18b2ce59c7d39993 namespace: kserve-ci-e2e-test resourceVersion: "13833" uid: c197c06b-27b3-45bc-9e6a-c75f896c7f30 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5 namespace: kserve-ci-e2e-test resourceVersion: "13468" uid: 35d8edba-6dc8-441f-b65f-9d4e667b4dcf kind: Event lastTimestamp: "2026-05-25T12:21:27Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-25T12:21:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5.18b2ce59c7e046e0 namespace: kserve-ci-e2e-test resourceVersion: "13834" uid: ac0a1ff8-695b-4820-a765-b65ebb4757c4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5 namespace: kserve-ci-e2e-test resourceVersion: "13468" uid: 35d8edba-6dc8-441f-b65f-9d4e667b4dcf kind: Event lastTimestamp: "2026-05-25T12:21:29Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.117s (2.117s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-25T12:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:29Z" name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5.18b2ce5a461d0b28 namespace: kserve-ci-e2e-test resourceVersion: "13852" uid: 0075c06b-8a55-4832-aecb-29f3106583cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5 namespace: kserve-ci-e2e-test resourceVersion: "13468" uid: 35d8edba-6dc8-441f-b65f-9d4e667b4dcf kind: Event lastTimestamp: "2026-05-25T12:21:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:29Z" name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5.18b2ce5a4a31e661 namespace: kserve-ci-e2e-test resourceVersion: "13854" uid: cd99ce20-220f-48a9-a3dc-44c2216f9ea0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5 namespace: kserve-ci-e2e-test resourceVersion: "13468" uid: 35d8edba-6dc8-441f-b65f-9d4e667b4dcf kind: Event lastTimestamp: "2026-05-25T12:21:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:21:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:29Z" name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5.18b2ce5a4aafe588 namespace: kserve-ci-e2e-test resourceVersion: "13855" uid: d5bfd492-8eb1-41fc-b744-a527fa0179c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T12:21:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5 namespace: kserve-ci-e2e-test resourceVersion: "13468" uid: 35d8edba-6dc8-441f-b65f-9d4e667b4dcf kind: Event lastTimestamp: "2026-05-25T12:22:17Z" message: 'Readiness probe failed: dial tcp 10.132.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:17Z" name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5.18b2ce5aa5b06723 namespace: kserve-ci-e2e-test resourceVersion: "14264" uid: 0ddb76d7-1b82-4b7f-abba-6f65496a228c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5 namespace: kserve-ci-e2e-test resourceVersion: "13468" uid: 35d8edba-6dc8-441f-b65f-9d4e667b4dcf kind: Event lastTimestamp: "2026-05-25T12:23:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:00Z" name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5.18b2ce6f7a4d8ad4 namespace: kserve-ci-e2e-test resourceVersion: "14898" uid: 4da438ff-c998-4178-819f-de5feb4b342c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5 namespace: kserve-ci-e2e-test resourceVersion: "13468" uid: 35d8edba-6dc8-441f-b65f-9d4e667b4dcf kind: Event lastTimestamp: "2026-05-25T12:23:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:00Z" name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5.18b2ce6f7a4e93f2 namespace: kserve-ci-e2e-test resourceVersion: "14896" uid: ab432efb-86fb-4f70-8a3e-d1a759e61ab1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5 namespace: kserve-ci-e2e-test resourceVersion: "13468" uid: 35d8edba-6dc8-441f-b65f-9d4e667b4dcf kind: Event lastTimestamp: "2026-05-25T12:23:02Z" message: 'Readiness probe failed: Get "https://10.132.0.17:8643/healthz": dial tcp 10.132.0.17:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:02Z" name: isvc-xgboost-graph-predictor-669d8d6456-x8tg5.18b2ce6fd5e0b8ac namespace: kserve-ci-e2e-test resourceVersion: "15030" uid: 321d4f8d-736b-4315-945f-7389365c4dd5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "13456" uid: 752d01a9-29e4-4471-a5b8-bebee0ac721f kind: Event lastTimestamp: "2026-05-25T12:21:01Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-x8tg5' metadata: creationTimestamp: "2026-05-25T12:21:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:21:01Z" name: isvc-xgboost-graph-predictor-669d8d6456.18b2ce53a0677f0e namespace: kserve-ci-e2e-test resourceVersion: "13467" uid: f484c0d6-0bb8-4312-aed1-2459ca007d7b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13455" uid: ce068dc1-a923-4815-9e7a-727b2686f681 kind: Event lastTimestamp: "2026-05-25T12:21:01Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:21:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:21:01Z" name: isvc-xgboost-graph-predictor.18b2ce5397bad3ac namespace: kserve-ci-e2e-test resourceVersion: "13458" uid: f2aaf742-1de1-4883-9a52-77246e57e444 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T12:21:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13457" uid: da7324e4-42f8-403f-8a74-b011cca9b9dc kind: Event lastTimestamp: "2026-05-25T12:22:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:21:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:01Z" name: isvc-xgboost-graph-predictor.18b2ce5716f569b1 namespace: kserve-ci-e2e-test resourceVersion: "14120" uid: 440d48f9-c6c5-4aba-8ae1-cc768afd7799 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T12:21:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13457" uid: da7324e4-42f8-403f-8a74-b011cca9b9dc kind: Event lastTimestamp: "2026-05-25T12:22:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:21:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:01Z" name: isvc-xgboost-graph-predictor.18b2ce57177ff66e namespace: kserve-ci-e2e-test resourceVersion: "14121" uid: 6367de04-47f6-4e1d-9165-22becce083e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:22:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13693" uid: da7324e4-42f8-403f-8a74-b011cca9b9dc kind: Event lastTimestamp: "2026-05-25T12:22:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:46Z" name: isvc-xgboost-graph-predictor.18b2ce6512bccf23 namespace: kserve-ci-e2e-test resourceVersion: "14702" uid: e099d598-fd70-4626-b1ac-7c40ce7a994b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:22:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13693" uid: da7324e4-42f8-403f-8a74-b011cca9b9dc kind: Event lastTimestamp: "2026-05-25T12:22:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:46Z" name: isvc-xgboost-graph-predictor.18b2ce65132b267c namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 0c283450-4273-426f-a197-3615a6ce9e15 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-25T12:21:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13442" uid: b72ac8b3-eaf6-4dff-837c-7fc5d0535236 kind: Event lastTimestamp: "2026-05-25T12:22:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:21:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:22:27Z" name: isvc-xgboost-graph.18b2ce5397f0e14e namespace: kserve-ci-e2e-test resourceVersion: "14396" uid: 150a5711-a136-4e0f-ba32-d15ad8cf96ae reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14393" uid: b72ac8b3-eaf6-4dff-837c-7fc5d0535236 kind: Event lastTimestamp: "2026-05-25T12:22:27Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-05-25T12:22:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:22:27Z" name: isvc-xgboost-graph.18b2ce67b5e65df7 namespace: kserve-ci-e2e-test resourceVersion: "14395" uid: c70771a8-246f-4014-be3c-1fc9e8aede98 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:22:50.770443Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-77c65d8d88-vkzr5 namespace: kserve-ci-e2e-test resourceVersion: "14761" uid: ee9e2f49-0644-46d5-9659-1d685023f00f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-77c65d8d88-vkzr5 to ip-10-0-143-107.ec2.internal metadata: creationTimestamp: "2026-05-25T12:22:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:22:50Z" name: model-chainer-77c65d8d88-vkzr5.18b2ce6d1b5aaa37 namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: aff493bb-cdfc-4d0c-bac9-244a0ecf50c6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:51Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-77c65d8d88-vkzr5 namespace: kserve-ci-e2e-test resourceVersion: "14770" uid: ee9e2f49-0644-46d5-9659-1d685023f00f kind: Event lastTimestamp: "2026-05-25T12:22:51Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:22:51Z" name: model-chainer-77c65d8d88-vkzr5.18b2ce6d34d351d5 namespace: kserve-ci-e2e-test resourceVersion: "14782" uid: 1c62070f-de28-41bf-b3f9-fc1c82e10dc1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-77c65d8d88-vkzr5 namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: ee9e2f49-0644-46d5-9659-1d685023f00f kind: Event lastTimestamp: "2026-05-25T12:22:51Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:41f0ca4b45114a1d81ffad66a667eba2baa4df82ad74c2ffc611dc9cdef0d94a" already present on machine metadata: creationTimestamp: "2026-05-25T12:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:51Z" name: model-chainer-77c65d8d88-vkzr5.18b2ce6d36599ab1 namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: c32e9c1b-7c1c-4357-8e6c-b8a1f2823d14 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-77c65d8d88-vkzr5 namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: ee9e2f49-0644-46d5-9659-1d685023f00f kind: Event lastTimestamp: "2026-05-25T12:22:51Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-05-25T12:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:51Z" name: model-chainer-77c65d8d88-vkzr5.18b2ce6d3920e295 namespace: kserve-ci-e2e-test resourceVersion: "14785" uid: a15a4a91-c6d8-4edf-ac53-0dbb1faea9d0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-77c65d8d88-vkzr5 namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: ee9e2f49-0644-46d5-9659-1d685023f00f kind: Event lastTimestamp: "2026-05-25T12:22:51Z" message: Started container model-chainer metadata: creationTimestamp: "2026-05-25T12:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:51Z" name: model-chainer-77c65d8d88-vkzr5.18b2ce6d39ad6232 namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: fca0bb3f-c8f2-43df-8d16-76ff300c751a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-77c65d8d88-vkzr5 namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: ee9e2f49-0644-46d5-9659-1d685023f00f kind: Event lastTimestamp: "2026-05-25T12:23:00Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-05-25T12:23:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:00Z" name: model-chainer-77c65d8d88-vkzr5.18b2ce6f74f22b24 namespace: kserve-ci-e2e-test resourceVersion: "14877" uid: f0034bc7-da95-4a2c-8377-96f299bcc332 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-77c65d8d88-vkzr5 namespace: kserve-ci-e2e-test resourceVersion: "14763" uid: ee9e2f49-0644-46d5-9659-1d685023f00f kind: Event lastTimestamp: "2026-05-25T12:23:27Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T12:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:27Z" name: model-chainer-77c65d8d88-vkzr5.18b2ce6fe3751374 namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: 67e82ff8-aa82-4c39-ab39-b77393cd1591 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-77c65d8d88 namespace: kserve-ci-e2e-test resourceVersion: "14757" uid: 689b3ac5-a249-4fbe-b84d-c27b87bd643a kind: Event lastTimestamp: "2026-05-25T12:22:50Z" message: 'Created pod: model-chainer-77c65d8d88-vkzr5' metadata: creationTimestamp: "2026-05-25T12:22:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:50Z" name: model-chainer-77c65d8d88.18b2ce6d1ab4646d namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: 48770bb3-4b64-4264-990a-696c32ce7ebe reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14755" uid: 064a9ec7-8f0e-4b93-a106-5bba1b902eab kind: Event lastTimestamp: "2026-05-25T12:22:50Z" message: Scaled up replica set model-chainer-77c65d8d88 from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:22:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:50Z" name: model-chainer.18b2ce6d19ef0559 namespace: kserve-ci-e2e-test resourceVersion: "14759" uid: f91972fc-f8fd-4a58-9191-965a87b7bdb6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 8fe7f572-a939-4e21-adcd-8a157d553003 kind: Event lastTimestamp: "2026-05-25T12:22:50Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-05-25T12:22:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:22:50Z" name: model-chainer.18b2ce6d1ec2c56a namespace: kserve-ci-e2e-test resourceVersion: "14774" uid: bd102a52-60c2-41ef-ae0e-af4b142e56d0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14747" uid: 8fe7f572-a939-4e21-adcd-8a157d553003 kind: Event lastTimestamp: "2026-05-25T12:22:50Z" message: 'Failed to update status for InferenceGraph "model-chainer": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:22:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:22:50Z" name: model-chainer.18b2ce6d217ad343 namespace: kserve-ci-e2e-test resourceVersion: "14777" uid: e6585606-1831-4514-8ade-17bda6990ec9 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14747" uid: 8fe7f572-a939-4e21-adcd-8a157d553003 kind: Event lastTimestamp: "2026-05-25T12:22:50Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:22:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:22:50Z" name: model-chainer.18b2ce6d217b3a95 namespace: kserve-ci-e2e-test resourceVersion: "14778" uid: b2248959-cd7f-426b-bedd-9e1f7aa29143 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:34:36.339430Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-4d75e-7f95b4f9f7-6b88l namespace: kserve-ci-e2e-test resourceVersion: "20946" uid: cce79cc8-362a-41f6-8a05-a6a5bd6b1fc7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-4d75e-7f95b4f9f7-6b88l to ip-10-0-143-107.ec2.internal metadata: creationTimestamp: "2026-05-25T12:34:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:34:36Z" name: sequence-graph-4d75e-7f95b4f9f7-6b88l.18b2cf11628b0679 namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: a934f0af-d9ac-4fa1-9e3b-bfaebc04381d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:34:36Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-4d75e-7f95b4f9f7-6b88l namespace: kserve-ci-e2e-test resourceVersion: "20949" uid: cce79cc8-362a-41f6-8a05-a6a5bd6b1fc7 kind: Event lastTimestamp: "2026-05-25T12:34:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-4d75e-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:34:36Z" name: sequence-graph-4d75e-7f95b4f9f7-6b88l.18b2cf116b643bd6 namespace: kserve-ci-e2e-test resourceVersion: "20967" uid: 17264fbc-224d-46ab-8639-4769c86e1d86 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:34:37Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-4d75e-7f95b4f9f7-6b88l namespace: kserve-ci-e2e-test resourceVersion: "20958" uid: cce79cc8-362a-41f6-8a05-a6a5bd6b1fc7 kind: Event lastTimestamp: "2026-05-25T12:34:37Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:34:37Z" name: sequence-graph-4d75e-7f95b4f9f7-6b88l.18b2cf119f4aac6c namespace: kserve-ci-e2e-test resourceVersion: "20974" uid: 8b26157d-1d48-413a-9b48-8d4f9c4b19f2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:34:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4d75e} kind: Pod name: sequence-graph-4d75e-7f95b4f9f7-6b88l namespace: kserve-ci-e2e-test resourceVersion: "20949" uid: cce79cc8-362a-41f6-8a05-a6a5bd6b1fc7 kind: Event lastTimestamp: "2026-05-25T12:34:37Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:41f0ca4b45114a1d81ffad66a667eba2baa4df82ad74c2ffc611dc9cdef0d94a" already present on machine metadata: creationTimestamp: "2026-05-25T12:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:34:37Z" name: sequence-graph-4d75e-7f95b4f9f7-6b88l.18b2cf11a0bc94de namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: ee039e13-ea49-40fd-868a-b97e48950ef8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:34:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4d75e} kind: Pod name: sequence-graph-4d75e-7f95b4f9f7-6b88l namespace: kserve-ci-e2e-test resourceVersion: "20949" uid: cce79cc8-362a-41f6-8a05-a6a5bd6b1fc7 kind: Event lastTimestamp: "2026-05-25T12:34:37Z" message: 'Created container: sequence-graph-4d75e' metadata: creationTimestamp: "2026-05-25T12:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:34:37Z" name: sequence-graph-4d75e-7f95b4f9f7-6b88l.18b2cf11a417b51b namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: 40efcdb7-c765-4977-bcc1-25d6e1179501 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:34:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4d75e} kind: Pod name: sequence-graph-4d75e-7f95b4f9f7-6b88l namespace: kserve-ci-e2e-test resourceVersion: "20949" uid: cce79cc8-362a-41f6-8a05-a6a5bd6b1fc7 kind: Event lastTimestamp: "2026-05-25T12:34:37Z" message: Started container sequence-graph-4d75e metadata: creationTimestamp: "2026-05-25T12:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:34:37Z" name: sequence-graph-4d75e-7f95b4f9f7-6b88l.18b2cf11a485e3c6 namespace: kserve-ci-e2e-test resourceVersion: "20979" uid: 52c5d7de-f882-401c-86a3-89367dbbdd87 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4d75e} kind: Pod name: sequence-graph-4d75e-7f95b4f9f7-6b88l namespace: kserve-ci-e2e-test resourceVersion: "20949" uid: cce79cc8-362a-41f6-8a05-a6a5bd6b1fc7 kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: Stopping container sequence-graph-4d75e metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:51Z" name: sequence-graph-4d75e-7f95b4f9f7-6b88l.18b2cf84907d4436 namespace: kserve-ci-e2e-test resourceVersion: "24535" uid: 81591b0f-5bed-43c1-84f8-9e4f8ba78133 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:42:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4d75e} kind: Pod name: sequence-graph-4d75e-7f95b4f9f7-6b88l namespace: kserve-ci-e2e-test resourceVersion: "20949" uid: cce79cc8-362a-41f6-8a05-a6a5bd6b1fc7 kind: Event lastTimestamp: "2026-05-25T12:43:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T12:42:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:18Z" name: sequence-graph-4d75e-7f95b4f9f7-6b88l.18b2cf852c00ea75 namespace: kserve-ci-e2e-test resourceVersion: "24902" uid: a7021cd9-0b5e-4017-888e-e6b7bffc2d05 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:34:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-4d75e-7f95b4f9f7 namespace: kserve-ci-e2e-test resourceVersion: "20943" uid: 10686273-385b-4345-9bda-e49d6fa0ee09 kind: Event lastTimestamp: "2026-05-25T12:34:36Z" message: 'Created pod: sequence-graph-4d75e-7f95b4f9f7-6b88l' metadata: creationTimestamp: "2026-05-25T12:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:34:36Z" name: sequence-graph-4d75e-7f95b4f9f7.18b2cf116218fbc3 namespace: kserve-ci-e2e-test resourceVersion: "20948" uid: 166bbf84-433f-4235-910c-855d3eaafcf7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:34:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-4d75e namespace: kserve-ci-e2e-test resourceVersion: "20941" uid: 798449ec-eb58-4ee7-ae34-f7ab2f3aad10 kind: Event lastTimestamp: "2026-05-25T12:34:36Z" message: Scaled up replica set sequence-graph-4d75e-7f95b4f9f7 from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:34:36Z" name: sequence-graph-4d75e.18b2cf11614468df namespace: kserve-ci-e2e-test resourceVersion: "20944" uid: 3cf256db-9519-4cac-a6c9-4b3ef52adb92 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:34:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-4d75e namespace: kserve-ci-e2e-test resourceVersion: "20960" uid: c57db6dc-0c1b-4f7a-86ba-c2cc113cbf2d kind: Event lastTimestamp: "2026-05-25T12:34:36Z" message: InferenceGraph [sequence-graph-4d75e] is Ready metadata: creationTimestamp: "2026-05-25T12:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:34:36Z" name: sequence-graph-4d75e.18b2cf11652eba7b namespace: kserve-ci-e2e-test resourceVersion: "20962" uid: d7fd13b2-27a5-458a-8bd8-1d6afa63f89a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:34:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-4d75e namespace: kserve-ci-e2e-test resourceVersion: "20942" uid: 3cd3667d-78db-45d1-a08b-e24efac06a08 kind: Event lastTimestamp: "2026-05-25T12:35:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:34:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:35:06Z" name: sequence-graph-4d75e.18b2cf14e0426055 namespace: kserve-ci-e2e-test resourceVersion: "21198" uid: 74fe159d-56a1-4ebc-b96f-0e6bbdac5390 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:34:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-4d75e namespace: kserve-ci-e2e-test resourceVersion: "20942" uid: 3cd3667d-78db-45d1-a08b-e24efac06a08 kind: Event lastTimestamp: "2026-05-25T12:35:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:34:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:35:06Z" name: sequence-graph-4d75e.18b2cf14e0c2e30a namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 7b409511-a2f5-4be4-b61c-721db1b0331e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:24:01.133483Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-833d6-8bd85f5d6-nq6xf namespace: kserve-ci-e2e-test resourceVersion: "15600" uid: 6aad06eb-8734-4662-91ef-21751d4c79a3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-833d6-8bd85f5d6-nq6xf to ip-10-0-143-107.ec2.internal metadata: creationTimestamp: "2026-05-25T12:24:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:24:01Z" name: sequence-graph-833d6-8bd85f5d6-nq6xf.18b2ce7d7d5169b0 namespace: kserve-ci-e2e-test resourceVersion: "15608" uid: 4ba71d69-3298-417a-927c-012f97fda73c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:24:01Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-833d6-8bd85f5d6-nq6xf namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 6aad06eb-8734-4662-91ef-21751d4c79a3 kind: Event lastTimestamp: "2026-05-25T12:24:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-833d6-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:24:01Z" name: sequence-graph-833d6-8bd85f5d6-nq6xf.18b2ce7d88c33860 namespace: kserve-ci-e2e-test resourceVersion: "15625" uid: 90152c27-a815-4076-b83b-629e371fa7c5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:24:02Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-833d6-8bd85f5d6-nq6xf namespace: kserve-ci-e2e-test resourceVersion: "15614" uid: 6aad06eb-8734-4662-91ef-21751d4c79a3 kind: Event lastTimestamp: "2026-05-25T12:24:02Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:24:02Z" name: sequence-graph-833d6-8bd85f5d6-nq6xf.18b2ce7dba4313f7 namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 265a2021-c0bd-4c02-a320-5cefd1683f8d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:24:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-833d6} kind: Pod name: sequence-graph-833d6-8bd85f5d6-nq6xf namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 6aad06eb-8734-4662-91ef-21751d4c79a3 kind: Event lastTimestamp: "2026-05-25T12:24:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:41f0ca4b45114a1d81ffad66a667eba2baa4df82ad74c2ffc611dc9cdef0d94a" already present on machine metadata: creationTimestamp: "2026-05-25T12:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:24:02Z" name: sequence-graph-833d6-8bd85f5d6-nq6xf.18b2ce7dbc472e6a namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: 4ad128d5-f17b-49b1-8f86-8f6f09eaa8f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:24:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-833d6} kind: Pod name: sequence-graph-833d6-8bd85f5d6-nq6xf namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 6aad06eb-8734-4662-91ef-21751d4c79a3 kind: Event lastTimestamp: "2026-05-25T12:24:02Z" message: 'Created container: sequence-graph-833d6' metadata: creationTimestamp: "2026-05-25T12:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:24:02Z" name: sequence-graph-833d6-8bd85f5d6-nq6xf.18b2ce7dbf07e3f4 namespace: kserve-ci-e2e-test resourceVersion: "15637" uid: 7bf7c7c4-1be5-4808-8bab-d94d60662c68 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:24:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-833d6} kind: Pod name: sequence-graph-833d6-8bd85f5d6-nq6xf namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 6aad06eb-8734-4662-91ef-21751d4c79a3 kind: Event lastTimestamp: "2026-05-25T12:24:02Z" message: Started container sequence-graph-833d6 metadata: creationTimestamp: "2026-05-25T12:24:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:24:02Z" name: sequence-graph-833d6-8bd85f5d6-nq6xf.18b2ce7dbf79f63e namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: f18460fb-e374-4ab5-b74d-feea8bb6fc85 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-833d6} kind: Pod name: sequence-graph-833d6-8bd85f5d6-nq6xf namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 6aad06eb-8734-4662-91ef-21751d4c79a3 kind: Event lastTimestamp: "2026-05-25T12:32:15Z" message: Stopping container sequence-graph-833d6 metadata: creationTimestamp: "2026-05-25T12:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:15Z" name: sequence-graph-833d6-8bd85f5d6-nq6xf.18b2cef0abb1a0ab namespace: kserve-ci-e2e-test resourceVersion: "19195" uid: 9e46e8bf-a3d1-4195-8264-563d892cb1f8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:32:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-833d6} kind: Pod name: sequence-graph-833d6-8bd85f5d6-nq6xf namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 6aad06eb-8734-4662-91ef-21751d4c79a3 kind: Event lastTimestamp: "2026-05-25T12:32:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T12:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:43Z" name: sequence-graph-833d6-8bd85f5d6-nq6xf.18b2cef163b371e6 namespace: kserve-ci-e2e-test resourceVersion: "19568" uid: 6a4ec0bd-72af-4872-85ca-ca534ada6dbf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:24:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-833d6-8bd85f5d6 namespace: kserve-ci-e2e-test resourceVersion: "15594" uid: bab50385-7d02-4222-b462-db435affa76f kind: Event lastTimestamp: "2026-05-25T12:24:01Z" message: 'Created pod: sequence-graph-833d6-8bd85f5d6-nq6xf' metadata: creationTimestamp: "2026-05-25T12:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:24:01Z" name: sequence-graph-833d6-8bd85f5d6.18b2ce7d7c64eaf1 namespace: kserve-ci-e2e-test resourceVersion: "15604" uid: 0d085573-4e0f-43e3-bddd-b048215811e1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:24:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-833d6 namespace: kserve-ci-e2e-test resourceVersion: "15592" uid: fabf0a3c-1bbd-43d2-9648-6a30d9aee533 kind: Event lastTimestamp: "2026-05-25T12:24:01Z" message: Scaled up replica set sequence-graph-833d6-8bd85f5d6 from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:24:01Z" name: sequence-graph-833d6.18b2ce7d7a59557e namespace: kserve-ci-e2e-test resourceVersion: "15595" uid: 07ba206e-02e9-480e-9138-70c4fa83a6f6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:24:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-833d6 namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: 3523ae69-6685-47a6-8c0c-a8fec4f08a5b kind: Event lastTimestamp: "2026-05-25T12:24:01Z" message: InferenceGraph [sequence-graph-833d6] is Ready metadata: creationTimestamp: "2026-05-25T12:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:24:01Z" name: sequence-graph-833d6.18b2ce7d82c10492 namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: a4272828-3e91-44c3-9453-646ddcc15166 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:24:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-833d6 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: 3523ae69-6685-47a6-8c0c-a8fec4f08a5b kind: Event lastTimestamp: "2026-05-25T12:24:01Z" message: 'Failed to update status for InferenceGraph "sequence-graph-833d6": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-833d6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:24:01Z" name: sequence-graph-833d6.18b2ce7d8761d4f7 namespace: kserve-ci-e2e-test resourceVersion: "15623" uid: 8dce0879-9974-4572-92c5-3f2f7173ef49 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:24:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-833d6 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: 3523ae69-6685-47a6-8c0c-a8fec4f08a5b kind: Event lastTimestamp: "2026-05-25T12:24:01Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-833d6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:24:01Z" name: sequence-graph-833d6.18b2ce7d876233e6 namespace: kserve-ci-e2e-test resourceVersion: "15624" uid: 03ae4b23-20a2-4b0e-83c3-d28a344ba67c reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:24:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-833d6 namespace: kserve-ci-e2e-test resourceVersion: "15593" uid: 055d0010-294f-46e2-8fdb-54275e3bb6af kind: Event lastTimestamp: "2026-05-25T12:24:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:24:46Z" name: sequence-graph-833d6.18b2ce80f987b8db namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 00757613-60e2-462a-b484-3f7e12df715f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:24:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-833d6 namespace: kserve-ci-e2e-test resourceVersion: "15593" uid: 055d0010-294f-46e2-8fdb-54275e3bb6af kind: Event lastTimestamp: "2026-05-25T12:24:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:24:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:24:46Z" name: sequence-graph-833d6.18b2ce80fa125b13 namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: 0ef7c2e5-8fef-4216-b32c-dc0bb8d8f78a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:33:26.029478Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-acbca-854dfbbf8-g5nv8 namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 0ef8b206-0424-407b-a392-ecc198007e76 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-acbca-854dfbbf8-g5nv8 to ip-10-0-143-107.ec2.internal metadata: creationTimestamp: "2026-05-25T12:33:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:33:26Z" name: sequence-graph-acbca-854dfbbf8-g5nv8.18b2cf0103be4aa9 namespace: kserve-ci-e2e-test resourceVersion: "20135" uid: 1f8ab0eb-16d0-4cdb-8560-535388e286e4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:26Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-acbca-854dfbbf8-g5nv8 namespace: kserve-ci-e2e-test resourceVersion: "20139" uid: 0ef8b206-0424-407b-a392-ecc198007e76 kind: Event lastTimestamp: "2026-05-25T12:33:26Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:33:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:33:26Z" name: sequence-graph-acbca-854dfbbf8-g5nv8.18b2cf011cf97697 namespace: kserve-ci-e2e-test resourceVersion: "20150" uid: 0e2eb184-55a4-481c-b903-6a4937d721f7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-acbca} kind: Pod name: sequence-graph-acbca-854dfbbf8-g5nv8 namespace: kserve-ci-e2e-test resourceVersion: "20132" uid: 0ef8b206-0424-407b-a392-ecc198007e76 kind: Event lastTimestamp: "2026-05-25T12:33:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:41f0ca4b45114a1d81ffad66a667eba2baa4df82ad74c2ffc611dc9cdef0d94a" already present on machine metadata: creationTimestamp: "2026-05-25T12:33:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:26Z" name: sequence-graph-acbca-854dfbbf8-g5nv8.18b2cf011ea93cad namespace: kserve-ci-e2e-test resourceVersion: "20152" uid: e0e3d5bb-3c2d-40a8-b781-cf4a2d007ebc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-acbca} kind: Pod name: sequence-graph-acbca-854dfbbf8-g5nv8 namespace: kserve-ci-e2e-test resourceVersion: "20132" uid: 0ef8b206-0424-407b-a392-ecc198007e76 kind: Event lastTimestamp: "2026-05-25T12:33:26Z" message: 'Created container: sequence-graph-acbca' metadata: creationTimestamp: "2026-05-25T12:33:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:26Z" name: sequence-graph-acbca-854dfbbf8-g5nv8.18b2cf012170f2da namespace: kserve-ci-e2e-test resourceVersion: "20156" uid: 643255e4-4a56-43ef-ac0a-64527535344e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-acbca} kind: Pod name: sequence-graph-acbca-854dfbbf8-g5nv8 namespace: kserve-ci-e2e-test resourceVersion: "20132" uid: 0ef8b206-0424-407b-a392-ecc198007e76 kind: Event lastTimestamp: "2026-05-25T12:33:26Z" message: Started container sequence-graph-acbca metadata: creationTimestamp: "2026-05-25T12:33:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:26Z" name: sequence-graph-acbca-854dfbbf8-g5nv8.18b2cf0121e6ee60 namespace: kserve-ci-e2e-test resourceVersion: "20157" uid: 1bb8d1ba-e7ad-46e7-b587-444ec6725a2a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-acbca} kind: Pod name: sequence-graph-acbca-854dfbbf8-g5nv8 namespace: kserve-ci-e2e-test resourceVersion: "20132" uid: 0ef8b206-0424-407b-a392-ecc198007e76 kind: Event lastTimestamp: "2026-05-25T12:33:36Z" message: Stopping container sequence-graph-acbca metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:36Z" name: sequence-graph-acbca-854dfbbf8-g5nv8.18b2cf035b43f42e namespace: kserve-ci-e2e-test resourceVersion: "20258" uid: c80b70c4-9333-4acc-9e84-5c5599ba3cfb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-acbca} kind: Pod name: sequence-graph-acbca-854dfbbf8-g5nv8 namespace: kserve-ci-e2e-test resourceVersion: "20132" uid: 0ef8b206-0424-407b-a392-ecc198007e76 kind: Event lastTimestamp: "2026-05-25T12:34:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T12:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:34:03Z" name: sequence-graph-acbca-854dfbbf8-g5nv8.18b2cf03e7d9c56f namespace: kserve-ci-e2e-test resourceVersion: "20621" uid: a0820b58-3343-4b4f-ae37-6ac15fa1aa92 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-acbca-854dfbbf8 namespace: kserve-ci-e2e-test resourceVersion: "20126" uid: 0e1b3342-cef8-4ff3-8e19-1c7c91a52fd6 kind: Event lastTimestamp: "2026-05-25T12:33:26Z" message: 'Created pod: sequence-graph-acbca-854dfbbf8-g5nv8' metadata: creationTimestamp: "2026-05-25T12:33:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:33:26Z" name: sequence-graph-acbca-854dfbbf8.18b2cf01033ceda0 namespace: kserve-ci-e2e-test resourceVersion: "20131" uid: 7a0e88bc-8a6d-4fa0-abb3-758f4a3b7a28 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-acbca namespace: kserve-ci-e2e-test resourceVersion: "20124" uid: 56286d0a-ce21-4668-a2a1-eda29c48a99a kind: Event lastTimestamp: "2026-05-25T12:33:25Z" message: Scaled up replica set sequence-graph-acbca-854dfbbf8 from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:33:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:33:26Z" name: sequence-graph-acbca.18b2cf0101ee5166 namespace: kserve-ci-e2e-test resourceVersion: "20127" uid: 3010fb00-7d97-4c79-ae02-f79721487df3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-acbca namespace: kserve-ci-e2e-test resourceVersion: "20141" uid: 0ba93409-ebb8-481f-a042-033ffb33c8e8 kind: Event lastTimestamp: "2026-05-25T12:33:26Z" message: InferenceGraph [sequence-graph-acbca] is Ready metadata: creationTimestamp: "2026-05-25T12:33:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:33:26Z" name: sequence-graph-acbca.18b2cf0106d20823 namespace: kserve-ci-e2e-test resourceVersion: "20143" uid: 0fd771f6-d426-4268-a428-fbfa1303dcc1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:44:31.628717Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-8902c-6b89df54cb-65tk5 namespace: kserve-ci-e2e-test resourceVersion: "25757" uid: dda39130-52e9-4d65-8fc4-731c4b3c7c6c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-8902c-6b89df54cb-65tk5 to ip-10-0-143-107.ec2.internal metadata: creationTimestamp: "2026-05-25T12:44:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:44:31Z" name: splitter-graph-8902c-6b89df54cb-65tk5.18b2cf9bfc8ca360 namespace: kserve-ci-e2e-test resourceVersion: "25762" uid: f46dbf59-5b6a-4cd2-931b-a16d7e71b93f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:31Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-8902c-6b89df54cb-65tk5 namespace: kserve-ci-e2e-test resourceVersion: "25759" uid: dda39130-52e9-4d65-8fc4-731c4b3c7c6c kind: Event lastTimestamp: "2026-05-25T12:44:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-8902c-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:44:31Z" name: splitter-graph-8902c-6b89df54cb-65tk5.18b2cf9c0474883c namespace: kserve-ci-e2e-test resourceVersion: "25776" uid: fe6d1a90-251a-4401-b560-0d6086b65efb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:32Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-8902c-6b89df54cb-65tk5 namespace: kserve-ci-e2e-test resourceVersion: "25766" uid: dda39130-52e9-4d65-8fc4-731c4b3c7c6c kind: Event lastTimestamp: "2026-05-25T12:44:32Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:44:32Z" name: splitter-graph-8902c-6b89df54cb-65tk5.18b2cf9c3927b1e4 namespace: kserve-ci-e2e-test resourceVersion: "25786" uid: 8ef04f3c-235b-40f1-841f-4e848b563800 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-8902c} kind: Pod name: splitter-graph-8902c-6b89df54cb-65tk5 namespace: kserve-ci-e2e-test resourceVersion: "25759" uid: dda39130-52e9-4d65-8fc4-731c4b3c7c6c kind: Event lastTimestamp: "2026-05-25T12:44:32Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:41f0ca4b45114a1d81ffad66a667eba2baa4df82ad74c2ffc611dc9cdef0d94a" already present on machine metadata: creationTimestamp: "2026-05-25T12:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:44:32Z" name: splitter-graph-8902c-6b89df54cb-65tk5.18b2cf9c3a7f6c30 namespace: kserve-ci-e2e-test resourceVersion: "25789" uid: 239307c5-28b7-4b08-b210-527047c6dd7d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-8902c} kind: Pod name: splitter-graph-8902c-6b89df54cb-65tk5 namespace: kserve-ci-e2e-test resourceVersion: "25759" uid: dda39130-52e9-4d65-8fc4-731c4b3c7c6c kind: Event lastTimestamp: "2026-05-25T12:44:32Z" message: 'Created container: splitter-graph-8902c' metadata: creationTimestamp: "2026-05-25T12:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:44:32Z" name: splitter-graph-8902c-6b89df54cb-65tk5.18b2cf9c3d0196d1 namespace: kserve-ci-e2e-test resourceVersion: "25790" uid: 76004cbe-9781-4fec-8287-ec1901ea3b63 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-8902c} kind: Pod name: splitter-graph-8902c-6b89df54cb-65tk5 namespace: kserve-ci-e2e-test resourceVersion: "25759" uid: dda39130-52e9-4d65-8fc4-731c4b3c7c6c kind: Event lastTimestamp: "2026-05-25T12:44:32Z" message: Started container splitter-graph-8902c metadata: creationTimestamp: "2026-05-25T12:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:44:32Z" name: splitter-graph-8902c-6b89df54cb-65tk5.18b2cf9c3d6ac810 namespace: kserve-ci-e2e-test resourceVersion: "25791" uid: 6f675a0c-d0e9-4c2e-8466-9f50a8c2d500 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:52:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-8902c} kind: Pod name: splitter-graph-8902c-6b89df54cb-65tk5 namespace: kserve-ci-e2e-test resourceVersion: "25759" uid: dda39130-52e9-4d65-8fc4-731c4b3c7c6c kind: Event lastTimestamp: "2026-05-25T12:52:46Z" message: Stopping container splitter-graph-8902c metadata: creationTimestamp: "2026-05-25T12:52:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:52:46Z" name: splitter-graph-8902c-6b89df54cb-65tk5.18b2d00f3070cde5 namespace: kserve-ci-e2e-test resourceVersion: "29509" uid: 817fa9f2-2f76-4a78-80cf-83138246ec0b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:52:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-8902c} kind: Pod name: splitter-graph-8902c-6b89df54cb-65tk5 namespace: kserve-ci-e2e-test resourceVersion: "25759" uid: dda39130-52e9-4d65-8fc4-731c4b3c7c6c kind: Event lastTimestamp: "2026-05-25T12:53:14Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T12:52:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:53:14Z" name: splitter-graph-8902c-6b89df54cb-65tk5.18b2d00fd750fbf9 namespace: kserve-ci-e2e-test resourceVersion: "29739" uid: 5e3eecee-5632-4ccc-984d-af214ec51d8e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-8902c-6b89df54cb namespace: kserve-ci-e2e-test resourceVersion: "25754" uid: 8d4e69ff-51c3-4660-bc2b-cb9059114e04 kind: Event lastTimestamp: "2026-05-25T12:44:31Z" message: 'Created pod: splitter-graph-8902c-6b89df54cb-65tk5' metadata: creationTimestamp: "2026-05-25T12:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:44:31Z" name: splitter-graph-8902c-6b89df54cb.18b2cf9bfc13d839 namespace: kserve-ci-e2e-test resourceVersion: "25758" uid: dd48bb88-393d-4e11-a060-2fb71c1f1184 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-8902c namespace: kserve-ci-e2e-test resourceVersion: "25750" uid: e0764364-4d7c-4b20-805a-806442d7f3f0 kind: Event lastTimestamp: "2026-05-25T12:44:31Z" message: Scaled up replica set splitter-graph-8902c-6b89df54cb from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:44:31Z" name: splitter-graph-8902c.18b2cf9bfb40c07a namespace: kserve-ci-e2e-test resourceVersion: "25755" uid: 70942d89-3d62-4714-813f-5451c3010ef2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-8902c namespace: kserve-ci-e2e-test resourceVersion: "25768" uid: e97ae037-dff9-4985-a44e-ec174dac151d kind: Event lastTimestamp: "2026-05-25T12:44:31Z" message: InferenceGraph [splitter-graph-8902c] is Ready metadata: creationTimestamp: "2026-05-25T12:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:44:31Z" name: splitter-graph-8902c.18b2cf9bff34bb9c namespace: kserve-ci-e2e-test resourceVersion: "25770" uid: a07d3b9f-a658-4e48-b72c-54c67b1632cb reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-8902c namespace: kserve-ci-e2e-test resourceVersion: "25743" uid: e97ae037-dff9-4985-a44e-ec174dac151d kind: Event lastTimestamp: "2026-05-25T12:44:31Z" message: 'Failed to update status for InferenceGraph "splitter-graph-8902c": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-8902c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:44:31Z" name: splitter-graph-8902c.18b2cf9c01b70e9d namespace: kserve-ci-e2e-test resourceVersion: "25771" uid: 91b29c8c-a58b-432a-b7c1-549ebccebd3f reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-8902c namespace: kserve-ci-e2e-test resourceVersion: "25743" uid: e97ae037-dff9-4985-a44e-ec174dac151d kind: Event lastTimestamp: "2026-05-25T12:44:31Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-8902c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:44:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:44:31Z" name: splitter-graph-8902c.18b2cf9c01b77c28 namespace: kserve-ci-e2e-test resourceVersion: "25773" uid: 95097dc7-6e32-47a8-8d65-8a296716ffa6 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:44:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-8902c namespace: kserve-ci-e2e-test resourceVersion: "25753" uid: b9f3a494-4184-4b3f-9d7a-5446b91c8664 kind: Event lastTimestamp: "2026-05-25T12:45:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:45:01Z" name: splitter-graph-8902c.18b2cf9f79d73f84 namespace: kserve-ci-e2e-test resourceVersion: "25999" uid: 1575c496-cea5-450b-b326-1ad03404d08b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:44:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-8902c namespace: kserve-ci-e2e-test resourceVersion: "25753" uid: b9f3a494-4184-4b3f-9d7a-5446b91c8664 kind: Event lastTimestamp: "2026-05-25T12:45:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:45:01Z" name: splitter-graph-8902c.18b2cf9f7a43256b namespace: kserve-ci-e2e-test resourceVersion: "26000" uid: f0af091e-4b3b-454c-ab9b-01c2d1349669 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:43:21.350815Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-fd4b4-74fd795bfc-rvr97 namespace: kserve-ci-e2e-test resourceVersion: "24942" uid: 7f71d6c2-9eed-4393-bb82-022261a41e08 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-fd4b4-74fd795bfc-rvr97 to ip-10-0-143-107.ec2.internal metadata: creationTimestamp: "2026-05-25T12:43:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:43:21Z" name: splitter-graph-fd4b4-74fd795bfc-rvr97.18b2cf8b9fa8f84d namespace: kserve-ci-e2e-test resourceVersion: "24948" uid: 6c37b58c-c771-4422-b7a7-9023b67a7366 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:21Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-fd4b4-74fd795bfc-rvr97 namespace: kserve-ci-e2e-test resourceVersion: "24951" uid: 7f71d6c2-9eed-4393-bb82-022261a41e08 kind: Event lastTimestamp: "2026-05-25T12:43:21Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:43:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:43:21Z" name: splitter-graph-fd4b4-74fd795bfc-rvr97.18b2cf8bb88065d9 namespace: kserve-ci-e2e-test resourceVersion: "24965" uid: 9621a6e6-3491-4a04-bab6-10191d8ca14d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-fd4b4} kind: Pod name: splitter-graph-fd4b4-74fd795bfc-rvr97 namespace: kserve-ci-e2e-test resourceVersion: "24944" uid: 7f71d6c2-9eed-4393-bb82-022261a41e08 kind: Event lastTimestamp: "2026-05-25T12:43:21Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:41f0ca4b45114a1d81ffad66a667eba2baa4df82ad74c2ffc611dc9cdef0d94a" already present on machine metadata: creationTimestamp: "2026-05-25T12:43:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:21Z" name: splitter-graph-fd4b4-74fd795bfc-rvr97.18b2cf8bba86aa52 namespace: kserve-ci-e2e-test resourceVersion: "24967" uid: 774a0371-b3ee-4c66-a258-f35d97d7c90a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-fd4b4} kind: Pod name: splitter-graph-fd4b4-74fd795bfc-rvr97 namespace: kserve-ci-e2e-test resourceVersion: "24944" uid: 7f71d6c2-9eed-4393-bb82-022261a41e08 kind: Event lastTimestamp: "2026-05-25T12:43:21Z" message: 'Created container: splitter-graph-fd4b4' metadata: creationTimestamp: "2026-05-25T12:43:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:21Z" name: splitter-graph-fd4b4-74fd795bfc-rvr97.18b2cf8bbd06d2c2 namespace: kserve-ci-e2e-test resourceVersion: "24968" uid: 11c22e94-b6c2-474d-b88c-8fef49522281 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-fd4b4} kind: Pod name: splitter-graph-fd4b4-74fd795bfc-rvr97 namespace: kserve-ci-e2e-test resourceVersion: "24944" uid: 7f71d6c2-9eed-4393-bb82-022261a41e08 kind: Event lastTimestamp: "2026-05-25T12:43:21Z" message: Started container splitter-graph-fd4b4 metadata: creationTimestamp: "2026-05-25T12:43:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:21Z" name: splitter-graph-fd4b4-74fd795bfc-rvr97.18b2cf8bbd84ab5c namespace: kserve-ci-e2e-test resourceVersion: "24969" uid: fd2de7e9-ab9d-4368-b228-6256af7b3058 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-fd4b4} kind: Pod name: splitter-graph-fd4b4-74fd795bfc-rvr97 namespace: kserve-ci-e2e-test resourceVersion: "24944" uid: 7f71d6c2-9eed-4393-bb82-022261a41e08 kind: Event lastTimestamp: "2026-05-25T12:43:31Z" message: Stopping container splitter-graph-fd4b4 metadata: creationTimestamp: "2026-05-25T12:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:31Z" name: splitter-graph-fd4b4-74fd795bfc-rvr97.18b2cf8df762532d namespace: kserve-ci-e2e-test resourceVersion: "25061" uid: 6b5476f8-59b6-44de-a394-2a862de4c247 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:43:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-fd4b4} kind: Pod name: splitter-graph-fd4b4-74fd795bfc-rvr97 namespace: kserve-ci-e2e-test resourceVersion: "24944" uid: 7f71d6c2-9eed-4393-bb82-022261a41e08 kind: Event lastTimestamp: "2026-05-25T12:43:58Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T12:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:58Z" name: splitter-graph-fd4b4-74fd795bfc-rvr97.18b2cf8e5806ae9c namespace: kserve-ci-e2e-test resourceVersion: "25419" uid: 10bcf40d-3da0-4ddb-b082-7c365725a57e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-fd4b4-74fd795bfc namespace: kserve-ci-e2e-test resourceVersion: "24939" uid: 314bd782-6933-4f84-ac6c-fbf2ac2ced8d kind: Event lastTimestamp: "2026-05-25T12:43:21Z" message: 'Created pod: splitter-graph-fd4b4-74fd795bfc-rvr97' metadata: creationTimestamp: "2026-05-25T12:43:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:43:21Z" name: splitter-graph-fd4b4-74fd795bfc.18b2cf8b9f17cbaa namespace: kserve-ci-e2e-test resourceVersion: "24943" uid: 8c6372eb-ff33-4271-b000-c45fc5cadbc9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-fd4b4 namespace: kserve-ci-e2e-test resourceVersion: "24937" uid: ed68549b-34f2-4932-af7f-56755ba0cec1 kind: Event lastTimestamp: "2026-05-25T12:43:21Z" message: Scaled up replica set splitter-graph-fd4b4-74fd795bfc from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:43:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:43:21Z" name: splitter-graph-fd4b4.18b2cf8b9e37c7e8 namespace: kserve-ci-e2e-test resourceVersion: "24940" uid: d78b153d-9bce-446d-8ec8-f8cfeadc6b6a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-fd4b4 namespace: kserve-ci-e2e-test resourceVersion: "24956" uid: 17eb1c0f-9de2-4790-ab5e-e3915be650d1 kind: Event lastTimestamp: "2026-05-25T12:43:21Z" message: InferenceGraph [splitter-graph-fd4b4] is Ready metadata: creationTimestamp: "2026-05-25T12:43:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:43:21Z" name: splitter-graph-fd4b4.18b2cf8ba405d5c5 namespace: kserve-ci-e2e-test resourceVersion: "24958" uid: 3e81e726-cf76-4e58-b90d-bd021cd35cf1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:42:51.183690Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-09598-predictor-587794bf84-zl6dc namespace: kserve-ci-e2e-test resourceVersion: "24571" uid: 7b0bcced-f9ba-4d60-a892-a604b11c68af kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-09598-predictor-587794bf84-zl6dc to ip-10-0-142-53.ec2.internal metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:42:51Z" name: success-200-isvc-09598-predictor-587794bf84-zl6dc.18b2cf84998f30ce namespace: kserve-ci-e2e-test resourceVersion: "24577" uid: d9763101-bc8d-4da1-a19f-e0cd9c52ed70 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-09598-predictor-587794bf84-zl6dc namespace: kserve-ci-e2e-test resourceVersion: "24575" uid: 7b0bcced-f9ba-4d60-a892-a604b11c68af kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-09598-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:51Z" name: success-200-isvc-09598-predictor-587794bf84-zl6dc.18b2cf84a6e956c6 namespace: kserve-ci-e2e-test resourceVersion: "24628" uid: 10f72aa2-66a0-4088-8ac9-313232208fcd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-09598-predictor-587794bf84-zl6dc namespace: kserve-ci-e2e-test resourceVersion: "24591" uid: 7b0bcced-f9ba-4d60-a892-a604b11c68af kind: Event lastTimestamp: "2026-05-25T12:42:52Z" message: Add eth0 [10.134.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:42:52Z" name: success-200-isvc-09598-predictor-587794bf84-zl6dc.18b2cf84d82999cb namespace: kserve-ci-e2e-test resourceVersion: "24654" uid: 5c158592-92db-43b0-ae94-d9bd442ba1af reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-09598-predictor-587794bf84-zl6dc namespace: kserve-ci-e2e-test resourceVersion: "24575" uid: 7b0bcced-f9ba-4d60-a892-a604b11c68af kind: Event lastTimestamp: "2026-05-25T12:42:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T12:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:52Z" name: success-200-isvc-09598-predictor-587794bf84-zl6dc.18b2cf84d9be37eb namespace: kserve-ci-e2e-test resourceVersion: "24656" uid: ddedef6d-5d74-4db6-bb4e-3bdc6167a8b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-09598-predictor-587794bf84-zl6dc namespace: kserve-ci-e2e-test resourceVersion: "24575" uid: 7b0bcced-f9ba-4d60-a892-a604b11c68af kind: Event lastTimestamp: "2026-05-25T12:42:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:52Z" name: success-200-isvc-09598-predictor-587794bf84-zl6dc.18b2cf84dcdc99a3 namespace: kserve-ci-e2e-test resourceVersion: "24658" uid: d8061fab-97c3-4378-b5d0-0c4b71b031c7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-09598-predictor-587794bf84-zl6dc namespace: kserve-ci-e2e-test resourceVersion: "24575" uid: 7b0bcced-f9ba-4d60-a892-a604b11c68af kind: Event lastTimestamp: "2026-05-25T12:42:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:52Z" name: success-200-isvc-09598-predictor-587794bf84-zl6dc.18b2cf84dd65abad namespace: kserve-ci-e2e-test resourceVersion: "24660" uid: 8a4dc7b9-5344-401c-894e-4215cf491291 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-09598-predictor-587794bf84-zl6dc namespace: kserve-ci-e2e-test resourceVersion: "24575" uid: 7b0bcced-f9ba-4d60-a892-a604b11c68af kind: Event lastTimestamp: "2026-05-25T12:42:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T12:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:52Z" name: success-200-isvc-09598-predictor-587794bf84-zl6dc.18b2cf84dd720e6e namespace: kserve-ci-e2e-test resourceVersion: "24661" uid: a024871a-ffdf-4fff-b2e0-6d438364a327 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-09598-predictor-587794bf84-zl6dc namespace: kserve-ci-e2e-test resourceVersion: "24575" uid: 7b0bcced-f9ba-4d60-a892-a604b11c68af kind: Event lastTimestamp: "2026-05-25T12:42:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:52Z" name: success-200-isvc-09598-predictor-587794bf84-zl6dc.18b2cf84e125ffa9 namespace: kserve-ci-e2e-test resourceVersion: "24663" uid: cc752158-257e-47c1-8ec4-525f4884afd7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-09598-predictor-587794bf84-zl6dc namespace: kserve-ci-e2e-test resourceVersion: "24575" uid: 7b0bcced-f9ba-4d60-a892-a604b11c68af kind: Event lastTimestamp: "2026-05-25T12:42:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:42:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:52Z" name: success-200-isvc-09598-predictor-587794bf84-zl6dc.18b2cf84e1e804bb namespace: kserve-ci-e2e-test resourceVersion: "24664" uid: e59559ff-c10c-4f9f-b4bc-e6e74eb99c0e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T12:42:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-09598-predictor-587794bf84-zl6dc namespace: kserve-ci-e2e-test resourceVersion: "24575" uid: 7b0bcced-f9ba-4d60-a892-a604b11c68af kind: Event lastTimestamp: "2026-05-25T12:43:39Z" message: 'Readiness probe failed: dial tcp 10.134.0.50:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:42:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:39Z" name: success-200-isvc-09598-predictor-587794bf84-zl6dc.18b2cf852465e49e namespace: kserve-ci-e2e-test resourceVersion: "25258" uid: 9024b913-1065-4707-a0bb-af67d57dad7e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:00:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-09598-predictor-587794bf84-zl6dc namespace: kserve-ci-e2e-test resourceVersion: "24575" uid: 7b0bcced-f9ba-4d60-a892-a604b11c68af kind: Event lastTimestamp: "2026-05-25T13:00:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T13:00:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:00:20Z" name: success-200-isvc-09598-predictor-587794bf84-zl6dc.18b2d078feabfa40 namespace: kserve-ci-e2e-test resourceVersion: "32676" uid: 70b4fc8b-2c0d-4f5e-8b7e-472c6ce521c4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:00:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-09598-predictor-587794bf84-zl6dc namespace: kserve-ci-e2e-test resourceVersion: "24575" uid: 7b0bcced-f9ba-4d60-a892-a604b11c68af kind: Event lastTimestamp: "2026-05-25T13:00:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T13:00:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:00:20Z" name: success-200-isvc-09598-predictor-587794bf84-zl6dc.18b2d078feac5875 namespace: kserve-ci-e2e-test resourceVersion: "32677" uid: 32e9c681-07b8-4313-9fbb-88c037fc5b52 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-09598-predictor-587794bf84 namespace: kserve-ci-e2e-test resourceVersion: "24559" uid: ecf914f2-41a6-487e-8dad-df29399f8595 kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: 'Created pod: success-200-isvc-09598-predictor-587794bf84-zl6dc' metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:42:51Z" name: success-200-isvc-09598-predictor-587794bf84.18b2cf84991dda02 namespace: kserve-ci-e2e-test resourceVersion: "24574" uid: 5222a29f-6ccc-4c40-8732-2db556d85972 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-09598-predictor namespace: kserve-ci-e2e-test resourceVersion: "24553" uid: dcd04c46-fcbc-405f-a1e8-18e03ccbda41 kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: Scaled up replica set success-200-isvc-09598-predictor-587794bf84 from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:42:51Z" name: success-200-isvc-09598-predictor.18b2cf849708e22d namespace: kserve-ci-e2e-test resourceVersion: "24561" uid: 505f4442-9dec-4dfa-8e69-108eacddfca5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:43:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-09598-predictor namespace: kserve-ci-e2e-test resourceVersion: "24558" uid: 722b6f92-7da3-4815-b4d2-ae70c07ce666 kind: Event lastTimestamp: "2026-05-25T12:43:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:43:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:43:21Z" name: success-200-isvc-09598-predictor.18b2cf8815e066ec namespace: kserve-ci-e2e-test resourceVersion: "24921" uid: f3abff2e-fc0c-433d-8754-5050305a8c3e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:43:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-09598-predictor namespace: kserve-ci-e2e-test resourceVersion: "24558" uid: 722b6f92-7da3-4815-b4d2-ae70c07ce666 kind: Event lastTimestamp: "2026-05-25T12:43:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:43:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:43:21Z" name: success-200-isvc-09598-predictor.18b2cf88165b8215 namespace: kserve-ci-e2e-test resourceVersion: "24923" uid: bbfd844a-9c2a-4fd8-abc6-407829d44086 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:43:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-09598-predictor namespace: kserve-ci-e2e-test resourceVersion: "24783" uid: 722b6f92-7da3-4815-b4d2-ae70c07ce666 kind: Event lastTimestamp: "2026-05-25T12:44:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:44:06Z" name: success-200-isvc-09598-predictor.18b2cf8f1488f87d namespace: kserve-ci-e2e-test resourceVersion: "25528" uid: 3a050502-fbd4-48ae-9c58-580bb6d1fe41 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:43:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-09598-predictor namespace: kserve-ci-e2e-test resourceVersion: "24783" uid: 722b6f92-7da3-4815-b4d2-ae70c07ce666 kind: Event lastTimestamp: "2026-05-25T12:44:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:44:06Z" name: success-200-isvc-09598-predictor.18b2cf8f1506f84b namespace: kserve-ci-e2e-test resourceVersion: "25529" uid: 6abfa64e-6a82-4d98-821f-1c0ae06bdca3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-09598 namespace: kserve-ci-e2e-test resourceVersion: "24537" uid: 30f96d57-6551-4520-8b3d-87c34261ded0 kind: Event lastTimestamp: "2026-05-25T12:43:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:43:49Z" name: success-200-isvc-09598.18b2cf8497144390 namespace: kserve-ci-e2e-test resourceVersion: "25354" uid: 83d4bede-c94e-495f-8a81-59df4264fd92 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-09598 namespace: kserve-ci-e2e-test resourceVersion: "24537" uid: 30f96d57-6551-4520-8b3d-87c34261ded0 kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: 'Failed to update status for InferenceService "success-200-isvc-09598": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-09598": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:42:51Z" name: success-200-isvc-09598.18b2cf8497e768cf namespace: kserve-ci-e2e-test resourceVersion: "24566" uid: ce7520d1-8dfe-4db2-9795-ad09c8673c60 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-09598 namespace: kserve-ci-e2e-test resourceVersion: "24537" uid: 30f96d57-6551-4520-8b3d-87c34261ded0 kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-09598": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:42:51Z" name: success-200-isvc-09598.18b2cf8497e7c58d namespace: kserve-ci-e2e-test resourceVersion: "24568" uid: 89b8c6e5-c706-4778-bff6-33da107f4141 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-09598 namespace: kserve-ci-e2e-test resourceVersion: "25351" uid: 30f96d57-6551-4520-8b3d-87c34261ded0 kind: Event lastTimestamp: "2026-05-25T12:43:49Z" message: InferenceService [success-200-isvc-09598] is Ready metadata: creationTimestamp: "2026-05-25T12:43:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:43:49Z" name: success-200-isvc-09598.18b2cf9234027eca namespace: kserve-ci-e2e-test resourceVersion: "25352" uid: 8196c1cb-a652-4ac1-97b7-93aa18af69e9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:32:56.371969Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql namespace: kserve-ci-e2e-test resourceVersion: "19761" uid: efcb96a7-a292-4595-880f-e7670a2d9928 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql to ip-10-0-142-53.ec2.internal metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:32:56Z" name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql.18b2cefa1c04946c namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: 3d6bda24-4254-4432-b5eb-8b445d28e7b1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: efcb96a7-a292-4595-880f-e7670a2d9928 kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:32:56Z" name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql.18b2cefa36583792 namespace: kserve-ci-e2e-test resourceVersion: "19829" uid: 789755e0-d04e-44a8-bd8b-91f66c0a7f79 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql namespace: kserve-ci-e2e-test resourceVersion: "19766" uid: efcb96a7-a292-4595-880f-e7670a2d9928 kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:56Z" name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql.18b2cefa3826dfa7 namespace: kserve-ci-e2e-test resourceVersion: "19832" uid: 2038f003-bf1c-4aa5-9bef-8225db29fe8c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql namespace: kserve-ci-e2e-test resourceVersion: "19766" uid: efcb96a7-a292-4595-880f-e7670a2d9928 kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:56Z" name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql.18b2cefa3be23aaf namespace: kserve-ci-e2e-test resourceVersion: "19835" uid: 582566da-de14-4641-bfda-99ef82f343a8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql namespace: kserve-ci-e2e-test resourceVersion: "19766" uid: efcb96a7-a292-4595-880f-e7670a2d9928 kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:56Z" name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql.18b2cefa3c62f5e1 namespace: kserve-ci-e2e-test resourceVersion: "19836" uid: fdc96f13-359d-4513-8400-47f15f7320e3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql namespace: kserve-ci-e2e-test resourceVersion: "19766" uid: efcb96a7-a292-4595-880f-e7670a2d9928 kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:56Z" name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql.18b2cefa3c767714 namespace: kserve-ci-e2e-test resourceVersion: "19837" uid: 43617f1d-4b43-491d-b75b-1b7905dc9184 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql namespace: kserve-ci-e2e-test resourceVersion: "19766" uid: efcb96a7-a292-4595-880f-e7670a2d9928 kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:56Z" name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql.18b2cefa40d935b1 namespace: kserve-ci-e2e-test resourceVersion: "19839" uid: 4af107be-3fec-4332-91fb-0ee69ce59f75 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql namespace: kserve-ci-e2e-test resourceVersion: "19766" uid: efcb96a7-a292-4595-880f-e7670a2d9928 kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:32:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:57Z" name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql.18b2cefa415fe637 namespace: kserve-ci-e2e-test resourceVersion: "19840" uid: 0f9259b3-44b1-43c8-bb14-10a4a1f7dea6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-25T12:32:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql namespace: kserve-ci-e2e-test resourceVersion: "19766" uid: efcb96a7-a292-4595-880f-e7670a2d9928 kind: Event lastTimestamp: "2026-05-25T12:42:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:32:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:24Z" name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql.18b2cefa8bd3201a namespace: kserve-ci-e2e-test resourceVersion: "24311" uid: 7f6dc12d-29e0-42a1-bbde-cb75494a262a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql namespace: kserve-ci-e2e-test resourceVersion: "19766" uid: efcb96a7-a292-4595-880f-e7670a2d9928 kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:21Z" name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql.18b2cf7d9dbd5a41 namespace: kserve-ci-e2e-test resourceVersion: "24194" uid: 60ff61f2-2155-47e6-ad65-bca11fbe03a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql namespace: kserve-ci-e2e-test resourceVersion: "19766" uid: efcb96a7-a292-4595-880f-e7670a2d9928 kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:21Z" name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql.18b2cf7d9dbe0432 namespace: kserve-ci-e2e-test resourceVersion: "24197" uid: 2c4923c8-321a-45c7-8163-e244491b5142 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql namespace: kserve-ci-e2e-test resourceVersion: "19766" uid: efcb96a7-a292-4595-880f-e7670a2d9928 kind: Event lastTimestamp: "2026-05-25T12:42:24Z" message: 'Readiness probe failed: Get "https://10.134.0.44:8643/healthz": dial tcp 10.134.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:42:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:24Z" name: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql.18b2cf7e5469491d namespace: kserve-ci-e2e-test resourceVersion: "24310" uid: b4f95984-13be-4539-9256-d8f64e0320b5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-155ee-predictor-6b9f6fd99b namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: 97758ad1-5c76-4290-b841-f05878187a0d kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: 'Created pod: success-200-isvc-155ee-predictor-6b9f6fd99b-qvjql' metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:32:56Z" name: success-200-isvc-155ee-predictor-6b9f6fd99b.18b2cefa1b73d5ab namespace: kserve-ci-e2e-test resourceVersion: "19765" uid: a1fdfbbd-a5c7-4f37-ac5c-6f6b7b5ecac0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-155ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "19746" uid: 134c13e0-6e52-438d-b5e4-f807f1fbe4b2 kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: Scaled up replica set success-200-isvc-155ee-predictor-6b9f6fd99b from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:32:56Z" name: success-200-isvc-155ee-predictor.18b2cefa197d0473 namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 46fce546-20ce-444b-95fa-74a06f56499e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:33:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-155ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: 473413d6-de8f-4523-9dc2-548d2a4906aa kind: Event lastTimestamp: "2026-05-25T12:33:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:33:41Z" name: success-200-isvc-155ee-predictor.18b2cefd986e7ae7 namespace: kserve-ci-e2e-test resourceVersion: "20428" uid: 195fe6a9-c652-42f5-98fa-2557d0d68f6a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:33:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-155ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: 473413d6-de8f-4523-9dc2-548d2a4906aa kind: Event lastTimestamp: "2026-05-25T12:33:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:33:41Z" name: success-200-isvc-155ee-predictor.18b2cefd990d4857 namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: e7ce3f87-c454-4237-9737-c73cac8eefab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:33:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-155ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "19991" uid: 473413d6-de8f-4523-9dc2-548d2a4906aa kind: Event lastTimestamp: "2026-05-25T12:34:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:34:26Z" name: success-200-isvc-155ee-predictor.18b2cf0815baf407 namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: 39a87620-905d-4f5e-b4d9-a5db3ea2f2e8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:33:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-155ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "19991" uid: 473413d6-de8f-4523-9dc2-548d2a4906aa kind: Event lastTimestamp: "2026-05-25T12:34:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:34:26Z" name: success-200-isvc-155ee-predictor.18b2cf0816419395 namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: da66d199-176a-410f-9576-676d5e5537c2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-155ee namespace: kserve-ci-e2e-test resourceVersion: "19738" uid: b9787adb-1a1f-4757-bc04-d5edf9c6bb81 kind: Event lastTimestamp: "2026-05-25T12:33:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:33:54Z" name: success-200-isvc-155ee.18b2cefa196206c6 namespace: kserve-ci-e2e-test resourceVersion: "20550" uid: fc06c848-77e8-478a-a06a-1314f57d4199 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-155ee namespace: kserve-ci-e2e-test resourceVersion: "19738" uid: b9787adb-1a1f-4757-bc04-d5edf9c6bb81 kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: 'Failed to update status for InferenceService "success-200-isvc-155ee": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-155ee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:32:56Z" name: success-200-isvc-155ee.18b2cefa1a056a18 namespace: kserve-ci-e2e-test resourceVersion: "19753" uid: 5e52a4cf-8ef6-467f-a1bb-f13f61d3c2ea reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-155ee namespace: kserve-ci-e2e-test resourceVersion: "19738" uid: b9787adb-1a1f-4757-bc04-d5edf9c6bb81 kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-155ee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:32:56Z" name: success-200-isvc-155ee.18b2cefa1a05bb8c namespace: kserve-ci-e2e-test resourceVersion: "19758" uid: 2a50c908-33ec-433d-ab4d-f08d94809d56 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-155ee namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: b9787adb-1a1f-4757-bc04-d5edf9c6bb81 kind: Event lastTimestamp: "2026-05-25T12:33:54Z" message: InferenceService [success-200-isvc-155ee] is Ready metadata: creationTimestamp: "2026-05-25T12:33:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:33:54Z" name: success-200-isvc-155ee.18b2cf079cade157 namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: 3269930b-caf7-402b-a3c3-072a04a5d414 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:21:01.062130Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf namespace: kserve-ci-e2e-test resourceVersion: "13420" uid: d1e41943-5b0a-4496-ba74-97cc2c381802 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf to ip-10-0-142-53.ec2.internal metadata: creationTimestamp: "2026-05-25T12:21:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:21:01Z" name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf.18b2ce53903a9eba namespace: kserve-ci-e2e-test resourceVersion: "13427" uid: 1bde093b-0264-48b7-8247-f6cc7bc37b9c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:21:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf namespace: kserve-ci-e2e-test resourceVersion: "13423" uid: d1e41943-5b0a-4496-ba74-97cc2c381802 kind: Event lastTimestamp: "2026-05-25T12:21:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-1742e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:21:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:01Z" name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf.18b2ce539a6ac073 namespace: kserve-ci-e2e-test resourceVersion: "13494" uid: eac69b66-84da-4c5d-88c4-afa600bc8f4e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf namespace: kserve-ci-e2e-test resourceVersion: "13437" uid: d1e41943-5b0a-4496-ba74-97cc2c381802 kind: Event lastTimestamp: "2026-05-25T12:21:03Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:21:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:21:03Z" name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf.18b2ce5404771ea8 namespace: kserve-ci-e2e-test resourceVersion: "13539" uid: 800427a7-3e57-4889-a5d8-5a6215fcb1c1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf namespace: kserve-ci-e2e-test resourceVersion: "13423" uid: d1e41943-5b0a-4496-ba74-97cc2c381802 kind: Event lastTimestamp: "2026-05-25T12:21:03Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" metadata: creationTimestamp: "2026-05-25T12:21:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:03Z" name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf.18b2ce54066089bb namespace: kserve-ci-e2e-test resourceVersion: "13541" uid: 06da52ec-2fa0-45e5-941e-25fa36e576d8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf namespace: kserve-ci-e2e-test resourceVersion: "13423" uid: d1e41943-5b0a-4496-ba74-97cc2c381802 kind: Event lastTimestamp: "2026-05-25T12:21:17Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" in 14.283s (14.283s including waiting). Image size: 1335729264 bytes.' metadata: creationTimestamp: "2026-05-25T12:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:17Z" name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf.18b2ce5759be3131 namespace: kserve-ci-e2e-test resourceVersion: "13709" uid: 78fac799-b0b4-4cd7-8015-72920c8fbc18 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf namespace: kserve-ci-e2e-test resourceVersion: "13423" uid: d1e41943-5b0a-4496-ba74-97cc2c381802 kind: Event lastTimestamp: "2026-05-25T12:21:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:17Z" name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf.18b2ce575f72c44e namespace: kserve-ci-e2e-test resourceVersion: "13716" uid: c0cd0a3c-9be4-431d-b832-5bef2be139c2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf namespace: kserve-ci-e2e-test resourceVersion: "13423" uid: d1e41943-5b0a-4496-ba74-97cc2c381802 kind: Event lastTimestamp: "2026-05-25T12:21:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:17Z" name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf.18b2ce575fe7f359 namespace: kserve-ci-e2e-test resourceVersion: "13720" uid: 07d89cb8-19d5-43da-9198-c6ec930bfabc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf namespace: kserve-ci-e2e-test resourceVersion: "13423" uid: d1e41943-5b0a-4496-ba74-97cc2c381802 kind: Event lastTimestamp: "2026-05-25T12:21:17Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-25T12:21:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:17Z" name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf.18b2ce575ff93306 namespace: kserve-ci-e2e-test resourceVersion: "13721" uid: 51eea2bf-5590-4df7-90b4-5669da1ddce7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf namespace: kserve-ci-e2e-test resourceVersion: "13423" uid: d1e41943-5b0a-4496-ba74-97cc2c381802 kind: Event lastTimestamp: "2026-05-25T12:21:19Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.559s (2.559s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-25T12:21:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:19Z" name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf.18b2ce57f885401d namespace: kserve-ci-e2e-test resourceVersion: "13740" uid: eed794ed-b4bf-49ac-a25e-2a93d1921400 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf namespace: kserve-ci-e2e-test resourceVersion: "13423" uid: d1e41943-5b0a-4496-ba74-97cc2c381802 kind: Event lastTimestamp: "2026-05-25T12:21:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:20Z" name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf.18b2ce57fc010ef9 namespace: kserve-ci-e2e-test resourceVersion: "13742" uid: 82c04661-e007-4868-a11a-a77fdeaa3c3a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf namespace: kserve-ci-e2e-test resourceVersion: "13423" uid: d1e41943-5b0a-4496-ba74-97cc2c381802 kind: Event lastTimestamp: "2026-05-25T12:21:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:20Z" name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf.18b2ce57fc7be298 namespace: kserve-ci-e2e-test resourceVersion: "13744" uid: 8fec8c7d-87e0-48a6-bbb6-9840d4b0b816 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:21:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf namespace: kserve-ci-e2e-test resourceVersion: "13423" uid: d1e41943-5b0a-4496-ba74-97cc2c381802 kind: Event lastTimestamp: "2026-05-25T12:21:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:21:57Z" name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf.18b2ce585db61e40 namespace: kserve-ci-e2e-test resourceVersion: "14089" uid: 16c5ef27-39fa-4cf2-920f-42e287995276 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf namespace: kserve-ci-e2e-test resourceVersion: "13423" uid: d1e41943-5b0a-4496-ba74-97cc2c381802 kind: Event lastTimestamp: "2026-05-25T12:22:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:22:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:30Z" name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf.18b2ce687fcb240a namespace: kserve-ci-e2e-test resourceVersion: "14484" uid: e6c6dbaa-0f48-40c1-8a8f-73909965eae1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf namespace: kserve-ci-e2e-test resourceVersion: "13423" uid: d1e41943-5b0a-4496-ba74-97cc2c381802 kind: Event lastTimestamp: "2026-05-25T12:22:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:22:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:30Z" name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf.18b2ce687fcb96db namespace: kserve-ci-e2e-test resourceVersion: "14488" uid: dc4734cf-878e-4ac7-970f-34809855100f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf namespace: kserve-ci-e2e-test resourceVersion: "13423" uid: d1e41943-5b0a-4496-ba74-97cc2c381802 kind: Event lastTimestamp: "2026-05-25T12:22:32Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:22:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:32Z" name: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf.18b2ce68e5d9cd43 namespace: kserve-ci-e2e-test resourceVersion: "14572" uid: 0251419d-bb26-4ab8-a8d1-62858f12c6ee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1742e-predictor-5bf4dfd9dd namespace: kserve-ci-e2e-test resourceVersion: "13406" uid: af023088-241e-48bb-9ee5-b7990fc319cd kind: Event lastTimestamp: "2026-05-25T12:21:01Z" message: 'Created pod: success-200-isvc-1742e-predictor-5bf4dfd9dd-8j4zf' metadata: creationTimestamp: "2026-05-25T12:21:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:21:01Z" name: success-200-isvc-1742e-predictor-5bf4dfd9dd.18b2ce538fbf2ea5 namespace: kserve-ci-e2e-test resourceVersion: "13422" uid: 6c397f2e-8735-4d22-af09-c41c67d2e0f3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:21:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1742e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13404" uid: ebb3606e-4710-4337-8b0d-532f069b3fcd kind: Event lastTimestamp: "2026-05-25T12:21:00Z" message: Scaled up replica set success-200-isvc-1742e-predictor-5bf4dfd9dd from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:21:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:21:00Z" name: success-200-isvc-1742e-predictor.18b2ce538a69fab2 namespace: kserve-ci-e2e-test resourceVersion: "13408" uid: 6cf7dd18-198f-4a60-94ca-9a9941818789 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T12:21:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1742e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13405" uid: 39ea4b22-af43-4123-a4f1-08605a677aac kind: Event lastTimestamp: "2026-05-25T12:22:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:01Z" name: success-200-isvc-1742e-predictor.18b2ce57097355cb namespace: kserve-ci-e2e-test resourceVersion: "14113" uid: a6da87d6-07a0-4710-b2d6-9c4242d35c68 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T12:21:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1742e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13405" uid: 39ea4b22-af43-4123-a4f1-08605a677aac kind: Event lastTimestamp: "2026-05-25T12:22:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:21:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:01Z" name: success-200-isvc-1742e-predictor.18b2ce570a157662 namespace: kserve-ci-e2e-test resourceVersion: "14114" uid: 7abe30dc-0b1e-491b-a354-d4cf5892e11d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1742e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13686" uid: 39ea4b22-af43-4123-a4f1-08605a677aac kind: Event lastTimestamp: "2026-05-25T12:22:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:16Z" name: success-200-isvc-1742e-predictor.18b2ce650544531e namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: b8ea4f2a-9fce-404e-94ad-0ec23a483322 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1742e-predictor namespace: kserve-ci-e2e-test resourceVersion: "13686" uid: 39ea4b22-af43-4123-a4f1-08605a677aac kind: Event lastTimestamp: "2026-05-25T12:22:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:16Z" name: success-200-isvc-1742e-predictor.18b2ce6505b6e832 namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: 21e336ba-3b40-421f-8214-a4ab36f798ae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:21:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1742e namespace: kserve-ci-e2e-test resourceVersion: "13389" uid: 85aeaccb-1e1d-40d2-ad17-d8ea94eeccf4 kind: Event lastTimestamp: "2026-05-25T12:22:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:21:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:22:07Z" name: success-200-isvc-1742e.18b2ce538a3dcb61 namespace: kserve-ci-e2e-test resourceVersion: "14185" uid: bad1cb27-2690-496d-8367-656b101ecda6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1742e namespace: kserve-ci-e2e-test resourceVersion: "14183" uid: 85aeaccb-1e1d-40d2-ad17-d8ea94eeccf4 kind: Event lastTimestamp: "2026-05-25T12:22:07Z" message: InferenceService [success-200-isvc-1742e] is Ready metadata: creationTimestamp: "2026-05-25T12:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:22:07Z" name: success-200-isvc-1742e.18b2ce63195105ee namespace: kserve-ci-e2e-test resourceVersion: "14184" uid: 1be8d359-feee-4c33-8b7a-6ce21d8657ca reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:22:31.026081Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n namespace: kserve-ci-e2e-test resourceVersion: "14495" uid: 7975c442-27c9-43e0-a7b5-5eba5e770eb3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-40bda-predictor-776fc95cd4-dr29n to ip-10-0-142-53.ec2.internal metadata: creationTimestamp: "2026-05-25T12:22:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:22:31Z" name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n.18b2ce68827f9bb3 namespace: kserve-ci-e2e-test resourceVersion: "14500" uid: b042def4-72f2-45e3-b1e0-7fb838631efb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 7975c442-27c9-43e0-a7b5-5eba5e770eb3 kind: Event lastTimestamp: "2026-05-25T12:22:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-40bda-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:31Z" name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n.18b2ce688b0ef127 namespace: kserve-ci-e2e-test resourceVersion: "14532" uid: c240c039-680d-4ae0-a6ee-99738fde2185 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: 7975c442-27c9-43e0-a7b5-5eba5e770eb3 kind: Event lastTimestamp: "2026-05-25T12:22:32Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:22:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:22:32Z" name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n.18b2ce68c0609e3c namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: 0fd150a9-6d59-4682-8564-73a2763abd77 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 7975c442-27c9-43e0-a7b5-5eba5e770eb3 kind: Event lastTimestamp: "2026-05-25T12:22:32Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T12:22:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:32Z" name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n.18b2ce68c22f94a4 namespace: kserve-ci-e2e-test resourceVersion: "14551" uid: 4fa6ab68-f48f-483c-af2e-4ce709f4fcf6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 7975c442-27c9-43e0-a7b5-5eba5e770eb3 kind: Event lastTimestamp: "2026-05-25T12:22:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:22:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:32Z" name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n.18b2ce68c58af011 namespace: kserve-ci-e2e-test resourceVersion: "14552" uid: e6ecea25-53f6-4be8-a998-4daffe68babc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 7975c442-27c9-43e0-a7b5-5eba5e770eb3 kind: Event lastTimestamp: "2026-05-25T12:22:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:22:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:32Z" name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n.18b2ce68c607d437 namespace: kserve-ci-e2e-test resourceVersion: "14554" uid: afeb2082-35c3-459d-8cf7-3b3778922036 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 7975c442-27c9-43e0-a7b5-5eba5e770eb3 kind: Event lastTimestamp: "2026-05-25T12:22:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T12:22:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:32Z" name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n.18b2ce68c6141694 namespace: kserve-ci-e2e-test resourceVersion: "14555" uid: 7d7d0be0-e8d2-40f6-bf50-eac6d8f776ee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 7975c442-27c9-43e0-a7b5-5eba5e770eb3 kind: Event lastTimestamp: "2026-05-25T12:22:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:22:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:32Z" name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n.18b2ce68c8deac55 namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: c3a33744-ffd2-49e3-a659-a0ce65d63724 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 7975c442-27c9-43e0-a7b5-5eba5e770eb3 kind: Event lastTimestamp: "2026-05-25T12:22:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:22:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:32Z" name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n.18b2ce68c953dcf6 namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 3259c4c2-a256-4b6c-be33-ea86f4784fe4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T12:22:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 7975c442-27c9-43e0-a7b5-5eba5e770eb3 kind: Event lastTimestamp: "2026-05-25T12:31:48Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:22:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:31:48Z" name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n.18b2ce68f521db25 namespace: kserve-ci-e2e-test resourceVersion: "18967" uid: 1e55c741-b5ed-4d9f-90c2-f005ecbf15d6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 7975c442-27c9-43e0-a7b5-5eba5e770eb3 kind: Event lastTimestamp: "2026-05-25T12:31:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:31:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:31:45Z" name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n.18b2cee9b4e440d9 namespace: kserve-ci-e2e-test resourceVersion: "18854" uid: 13d5a485-94f0-455e-9024-79eb5968601a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 7975c442-27c9-43e0-a7b5-5eba5e770eb3 kind: Event lastTimestamp: "2026-05-25T12:31:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:31:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:31:45Z" name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n.18b2cee9b4e4e93f namespace: kserve-ci-e2e-test resourceVersion: "18855" uid: 209ebcd1-5964-4b9c-81f4-9703cabf58b3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 7975c442-27c9-43e0-a7b5-5eba5e770eb3 kind: Event lastTimestamp: "2026-05-25T12:31:48Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:31:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:31:48Z" name: success-200-isvc-40bda-predictor-776fc95cd4-dr29n.18b2ceea6975527d namespace: kserve-ci-e2e-test resourceVersion: "18966" uid: c865cfbc-b651-4221-9cf4-fe6b5bf7fc93 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-40bda-predictor-776fc95cd4 namespace: kserve-ci-e2e-test resourceVersion: "14486" uid: be140175-6aa7-46d7-8c67-0923c61daa53 kind: Event lastTimestamp: "2026-05-25T12:22:31Z" message: 'Created pod: success-200-isvc-40bda-predictor-776fc95cd4-dr29n' metadata: creationTimestamp: "2026-05-25T12:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:31Z" name: success-200-isvc-40bda-predictor-776fc95cd4.18b2ce68820d0f57 namespace: kserve-ci-e2e-test resourceVersion: "14496" uid: 480d2460-9413-4406-b0e2-d47b94f4b0f8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-40bda-predictor namespace: kserve-ci-e2e-test resourceVersion: "14482" uid: ba1090d2-4979-463f-8e11-4f89f97485f9 kind: Event lastTimestamp: "2026-05-25T12:22:30Z" message: Scaled up replica set success-200-isvc-40bda-predictor-776fc95cd4 from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:22:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:30Z" name: success-200-isvc-40bda-predictor.18b2ce68807e78a1 namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: dd52056c-62d3-4f20-94d3-9be97c82d112 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-40bda-predictor namespace: kserve-ci-e2e-test resourceVersion: "14485" uid: c8696da2-70e2-471b-9305-6558c2bc9300 kind: Event lastTimestamp: "2026-05-25T12:22:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:22:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:46Z" name: success-200-isvc-40bda-predictor.18b2ce6bff224a0f namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: b5727fa1-2f3d-4ed6-b07d-3b039feae7b5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-40bda-predictor namespace: kserve-ci-e2e-test resourceVersion: "14485" uid: c8696da2-70e2-471b-9305-6558c2bc9300 kind: Event lastTimestamp: "2026-05-25T12:22:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:22:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:46Z" name: success-200-isvc-40bda-predictor.18b2ce6bff842f12 namespace: kserve-ci-e2e-test resourceVersion: "14698" uid: 6190669f-45b3-4e51-a4c1-09005162a3c8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:23:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-40bda-predictor namespace: kserve-ci-e2e-test resourceVersion: "14696" uid: c8696da2-70e2-471b-9305-6558c2bc9300 kind: Event lastTimestamp: "2026-05-25T12:24:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:24:01Z" name: success-200-isvc-40bda-predictor.18b2ce6f7e4f6c08 namespace: kserve-ci-e2e-test resourceVersion: "15598" uid: d29192f1-571c-4698-bf0e-c50f05b42fca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:23:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-40bda-predictor namespace: kserve-ci-e2e-test resourceVersion: "14696" uid: c8696da2-70e2-471b-9305-6558c2bc9300 kind: Event lastTimestamp: "2026-05-25T12:24:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:24:01Z" name: success-200-isvc-40bda-predictor.18b2ce6f7eb816d7 namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 66da0733-ec37-4a04-a26a-cb70d515df37 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:22:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-40bda namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 851120a4-eb5d-4c31-901b-45d26ae40050 kind: Event lastTimestamp: "2026-05-25T12:23:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:22:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:23:19Z" name: success-200-isvc-40bda.18b2ce688084f2b7 namespace: kserve-ci-e2e-test resourceVersion: "15202" uid: 06b5758f-9bdf-42fb-99e7-42adc048f6b5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-40bda namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 851120a4-eb5d-4c31-901b-45d26ae40050 kind: Event lastTimestamp: "2026-05-25T12:22:31Z" message: 'Failed to update status for InferenceService "success-200-isvc-40bda": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-40bda": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:22:31Z" name: success-200-isvc-40bda.18b2ce68810003c0 namespace: kserve-ci-e2e-test resourceVersion: "14492" uid: c160b35e-bcb5-4236-9e8a-9e850e7f81ed reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-40bda namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 851120a4-eb5d-4c31-901b-45d26ae40050 kind: Event lastTimestamp: "2026-05-25T12:22:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-40bda": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:22:31Z" name: success-200-isvc-40bda.18b2ce68810056be namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: 8b3f3665-e6bf-4597-a3f8-f098bb849a60 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-40bda namespace: kserve-ci-e2e-test resourceVersion: "15198" uid: 851120a4-eb5d-4c31-901b-45d26ae40050 kind: Event lastTimestamp: "2026-05-25T12:23:19Z" message: InferenceService [success-200-isvc-40bda] is Ready metadata: creationTimestamp: "2026-05-25T12:23:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:23:19Z" name: success-200-isvc-40bda.18b2ce73b0dd21f7 namespace: kserve-ci-e2e-test resourceVersion: "15199" uid: 5ed67883-0297-4f23-a0fc-c7e3a154474f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:33:36.295053Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw namespace: kserve-ci-e2e-test resourceVersion: "20304" uid: b5232f59-357c-4a8a-a062-13d1161f8a81 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4d75e-predictor-98c496fb-m8ddw to ip-10-0-142-53.ec2.internal metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:33:36Z" name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw.18b2cf03679e8374 namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: bdb732f3-cda5-490c-a72b-9cfa76e3b937 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: b5232f59-357c-4a8a-a062-13d1161f8a81 kind: Event lastTimestamp: "2026-05-25T12:33:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-4d75e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:36Z" name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw.18b2cf0373b4d101 namespace: kserve-ci-e2e-test resourceVersion: "20347" uid: 1aa36c31-e0a2-4153-a17a-1f9dd4afc8bb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw namespace: kserve-ci-e2e-test resourceVersion: "20316" uid: b5232f59-357c-4a8a-a062-13d1161f8a81 kind: Event lastTimestamp: "2026-05-25T12:33:37Z" message: Add eth0 [10.134.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:33:37Z" name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw.18b2cf03a5e864ff namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: 7522cf7b-cd5f-49f9-9b88-6ecccb0852e6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: b5232f59-357c-4a8a-a062-13d1161f8a81 kind: Event lastTimestamp: "2026-05-25T12:33:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T12:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:37Z" name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw.18b2cf03a798f5da namespace: kserve-ci-e2e-test resourceVersion: "20371" uid: bef5786b-2763-4275-b315-adaeb534fe22 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: b5232f59-357c-4a8a-a062-13d1161f8a81 kind: Event lastTimestamp: "2026-05-25T12:33:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:37Z" name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw.18b2cf03abc9c815 namespace: kserve-ci-e2e-test resourceVersion: "20374" uid: 9d60c7c0-f87d-443d-a725-f663b40603b2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: b5232f59-357c-4a8a-a062-13d1161f8a81 kind: Event lastTimestamp: "2026-05-25T12:33:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:37Z" name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw.18b2cf03ac43b3f1 namespace: kserve-ci-e2e-test resourceVersion: "20375" uid: 15375732-6902-4112-b76c-e023a8ed4227 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: b5232f59-357c-4a8a-a062-13d1161f8a81 kind: Event lastTimestamp: "2026-05-25T12:33:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T12:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:37Z" name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw.18b2cf03ac58b4d2 namespace: kserve-ci-e2e-test resourceVersion: "20378" uid: 5aa37382-37af-45ba-8855-237e132aa8be reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: b5232f59-357c-4a8a-a062-13d1161f8a81 kind: Event lastTimestamp: "2026-05-25T12:33:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:37Z" name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw.18b2cf03af8fb8e2 namespace: kserve-ci-e2e-test resourceVersion: "20379" uid: fb52f6ee-5330-46de-ad50-feeb2c1b4bce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: b5232f59-357c-4a8a-a062-13d1161f8a81 kind: Event lastTimestamp: "2026-05-25T12:33:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:37Z" name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw.18b2cf03b0187e21 namespace: kserve-ci-e2e-test resourceVersion: "20380" uid: 80334378-7fc4-49f0-8332-af18a65341a0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T12:33:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: b5232f59-357c-4a8a-a062-13d1161f8a81 kind: Event lastTimestamp: "2026-05-25T12:34:15Z" message: 'Readiness probe failed: dial tcp 10.134.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:33:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:34:15Z" name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw.18b2cf03e73cd6a3 namespace: kserve-ci-e2e-test resourceVersion: "20757" uid: ccf11474-c2f7-4865-b2a5-aac014e36155 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: b5232f59-357c-4a8a-a062-13d1161f8a81 kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:51Z" name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw.18b2cf84985f8b2d namespace: kserve-ci-e2e-test resourceVersion: "24569" uid: 150b1e72-d6b3-4ba3-bec8-37a270be72f9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: b5232f59-357c-4a8a-a062-13d1161f8a81 kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:42:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:51Z" name: success-200-isvc-4d75e-predictor-98c496fb-m8ddw.18b2cf849860332f namespace: kserve-ci-e2e-test resourceVersion: "24570" uid: a8360bbc-3541-4f06-8c2d-93c8d5b578e1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4d75e-predictor-98c496fb namespace: kserve-ci-e2e-test resourceVersion: "20289" uid: 4ef10c15-eaae-4664-ac45-61e6db2a7a88 kind: Event lastTimestamp: "2026-05-25T12:33:36Z" message: 'Created pod: success-200-isvc-4d75e-predictor-98c496fb-m8ddw' metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:33:36Z" name: success-200-isvc-4d75e-predictor-98c496fb.18b2cf036716aa1a namespace: kserve-ci-e2e-test resourceVersion: "20307" uid: c39f9336-2c7b-44a5-b0f0-1dff0204270e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4d75e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20283" uid: 506cd573-213c-45d5-bc57-f9eaf995ea4b kind: Event lastTimestamp: "2026-05-25T12:33:36Z" message: Scaled up replica set success-200-isvc-4d75e-predictor-98c496fb from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:33:36Z" name: success-200-isvc-4d75e-predictor.18b2cf03649ed037 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 95968f50-7c6e-41eb-b592-cea69e043504 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:33:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4d75e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20288" uid: 1945ca69-9d85-4f02-b15c-81578cd02311 kind: Event lastTimestamp: "2026-05-25T12:34:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:34:06Z" name: success-200-isvc-4d75e-predictor.18b2cf06e3c540e8 namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: e2f55ff4-6542-40c3-9e2c-4c03144f7d9b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:33:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4d75e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20288" uid: 1945ca69-9d85-4f02-b15c-81578cd02311 kind: Event lastTimestamp: "2026-05-25T12:34:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:34:06Z" name: success-200-isvc-4d75e-predictor.18b2cf06e44bd3f6 namespace: kserve-ci-e2e-test resourceVersion: "20649" uid: 8fba2d7e-ec03-409b-8eab-b28b0db0617c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:34:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4d75e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20503" uid: 1945ca69-9d85-4f02-b15c-81578cd02311 kind: Event lastTimestamp: "2026-05-25T12:34:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:34:51Z" name: success-200-isvc-4d75e-predictor.18b2cf0de2b2a4ea namespace: kserve-ci-e2e-test resourceVersion: "21083" uid: ea1371bb-d812-41bd-bf8d-9b86117ae4a6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:34:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4d75e-predictor namespace: kserve-ci-e2e-test resourceVersion: "20503" uid: 1945ca69-9d85-4f02-b15c-81578cd02311 kind: Event lastTimestamp: "2026-05-25T12:34:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:34:51Z" name: success-200-isvc-4d75e-predictor.18b2cf0de33c7098 namespace: kserve-ci-e2e-test resourceVersion: "21084" uid: b74efd05-181c-45b2-9cb6-ac96e1edb6a2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4d75e namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: e8ac9862-3ca5-42b1-b61e-9ecee9c07a1c kind: Event lastTimestamp: "2026-05-25T12:34:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:34:25Z" name: success-200-isvc-4d75e.18b2cf03646361b8 namespace: kserve-ci-e2e-test resourceVersion: "20853" uid: 4c2bb28a-8482-4c1d-a5b5-5ba3602befb2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4d75e namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: e8ac9862-3ca5-42b1-b61e-9ecee9c07a1c kind: Event lastTimestamp: "2026-05-25T12:33:36Z" message: 'Failed to update status for InferenceService "success-200-isvc-4d75e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4d75e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:33:36Z" name: success-200-isvc-4d75e.18b2cf0365485e12 namespace: kserve-ci-e2e-test resourceVersion: "20299" uid: ea1e72d5-8957-447b-a4c3-4af439043422 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4d75e namespace: kserve-ci-e2e-test resourceVersion: "20261" uid: e8ac9862-3ca5-42b1-b61e-9ecee9c07a1c kind: Event lastTimestamp: "2026-05-25T12:33:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4d75e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:33:36Z" name: success-200-isvc-4d75e.18b2cf036548bbc6 namespace: kserve-ci-e2e-test resourceVersion: "20301" uid: 81855289-6c23-4830-86b7-8834f5eed71d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:34:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4d75e namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: e8ac9862-3ca5-42b1-b61e-9ecee9c07a1c kind: Event lastTimestamp: "2026-05-25T12:34:25Z" message: InferenceService [success-200-isvc-4d75e] is Ready metadata: creationTimestamp: "2026-05-25T12:34:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:34:25Z" name: success-200-isvc-4d75e.18b2cf0ee1886ce0 namespace: kserve-ci-e2e-test resourceVersion: "20852" uid: b9a35780-554a-4a79-a438-bbf08daf76f9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:23:01.055012Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6 namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 6264ef64-b6b2-43bd-bcfb-f281e7205194 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6 to ip-10-0-142-53.ec2.internal metadata: creationTimestamp: "2026-05-25T12:23:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:23:01Z" name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6.18b2ce6f805cb0c6 namespace: kserve-ci-e2e-test resourceVersion: "14950" uid: 46237a87-3434-413b-8d37-ab5dcbd671fd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6 namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 6264ef64-b6b2-43bd-bcfb-f281e7205194 kind: Event lastTimestamp: "2026-05-25T12:23:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-833d6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:01Z" name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6.18b2ce6f8bce9b03 namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: d8d7f7a7-eb33-45f4-9305-6f5a1296de37 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6 namespace: kserve-ci-e2e-test resourceVersion: "14959" uid: 6264ef64-b6b2-43bd-bcfb-f281e7205194 kind: Event lastTimestamp: "2026-05-25T12:23:02Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:23:02Z" name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6.18b2ce6fbe6f238c namespace: kserve-ci-e2e-test resourceVersion: "15011" uid: 34ca9556-d2aa-44fb-aa83-625d84db6918 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6 namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 6264ef64-b6b2-43bd-bcfb-f281e7205194 kind: Event lastTimestamp: "2026-05-25T12:23:02Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T12:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:02Z" name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6.18b2ce6fc0212e47 namespace: kserve-ci-e2e-test resourceVersion: "15013" uid: 056a2430-d130-447d-baf5-7624fa4688f9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6 namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 6264ef64-b6b2-43bd-bcfb-f281e7205194 kind: Event lastTimestamp: "2026-05-25T12:23:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:02Z" name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6.18b2ce6fc4ffd10f namespace: kserve-ci-e2e-test resourceVersion: "15015" uid: 67731360-fcfb-4b6c-8801-5286693a61cb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6 namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 6264ef64-b6b2-43bd-bcfb-f281e7205194 kind: Event lastTimestamp: "2026-05-25T12:23:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:02Z" name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6.18b2ce6fc578d56f namespace: kserve-ci-e2e-test resourceVersion: "15016" uid: 7a0d6895-c1a1-4a19-9804-d1ef0c7bba07 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6 namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 6264ef64-b6b2-43bd-bcfb-f281e7205194 kind: Event lastTimestamp: "2026-05-25T12:23:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T12:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:02Z" name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6.18b2ce6fc585a83e namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 6e3bce29-c6d4-4aec-9f94-b2fb9deecf5b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6 namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 6264ef64-b6b2-43bd-bcfb-f281e7205194 kind: Event lastTimestamp: "2026-05-25T12:23:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:02Z" name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6.18b2ce6fc8b7f071 namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: 5d7965d1-a75f-4a5e-94b0-e06a49a0f5ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6 namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 6264ef64-b6b2-43bd-bcfb-f281e7205194 kind: Event lastTimestamp: "2026-05-25T12:23:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:23:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:02Z" name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6.18b2ce6fc93726ca namespace: kserve-ci-e2e-test resourceVersion: "15021" uid: b8c2899d-2f35-4e18-8bfb-daf289679252 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T12:23:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6 namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 6264ef64-b6b2-43bd-bcfb-f281e7205194 kind: Event lastTimestamp: "2026-05-25T12:32:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:19Z" name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6.18b2ce6ff828c172 namespace: kserve-ci-e2e-test resourceVersion: "19350" uid: 29ba320f-7ce5-4ceb-a4a2-33b5716e5f0d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6 namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 6264ef64-b6b2-43bd-bcfb-f281e7205194 kind: Event lastTimestamp: "2026-05-25T12:32:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:15Z" name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6.18b2cef0b3ab3db1 namespace: kserve-ci-e2e-test resourceVersion: "19229" uid: 8b0f167b-0f4d-4b54-a17b-a55dc7c503e5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6 namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 6264ef64-b6b2-43bd-bcfb-f281e7205194 kind: Event lastTimestamp: "2026-05-25T12:32:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:15Z" name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6.18b2cef0b3abd5ba namespace: kserve-ci-e2e-test resourceVersion: "19232" uid: d7bea96c-baa4-4f05-9e8f-618714a0b1fc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6 namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 6264ef64-b6b2-43bd-bcfb-f281e7205194 kind: Event lastTimestamp: "2026-05-25T12:32:19Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:32:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:19Z" name: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6.18b2cef16ca78166 namespace: kserve-ci-e2e-test resourceVersion: "19349" uid: c269b170-d127-4fbc-8a4f-8e90052d69dc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-833d6-predictor-65cfc8fdbb namespace: kserve-ci-e2e-test resourceVersion: "14922" uid: 74fa53b8-252a-445b-a3f7-f827d62b5987 kind: Event lastTimestamp: "2026-05-25T12:23:01Z" message: 'Created pod: success-200-isvc-833d6-predictor-65cfc8fdbb-xw7b6' metadata: creationTimestamp: "2026-05-25T12:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:23:01Z" name: success-200-isvc-833d6-predictor-65cfc8fdbb.18b2ce6f7ff23f62 namespace: kserve-ci-e2e-test resourceVersion: "14947" uid: 9c77df5e-d63a-4856-b195-bfc00db8eb7b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-833d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: a00c64c8-56ab-4a82-bf58-c503d1b454b6 kind: Event lastTimestamp: "2026-05-25T12:23:01Z" message: Scaled up replica set success-200-isvc-833d6-predictor-65cfc8fdbb from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:23:01Z" name: success-200-isvc-833d6-predictor.18b2ce6f7de4e60e namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: a6a28d67-c3a7-4133-9faf-2f5cbd948f5a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:23:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-833d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 18f7b2d8-85c4-41b2-b5ca-ac089bf973b0 kind: Event lastTimestamp: "2026-05-25T12:23:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:23:46Z" name: success-200-isvc-833d6-predictor.18b2ce72fcd1afdb namespace: kserve-ci-e2e-test resourceVersion: "15456" uid: dbe7acec-80ba-4d0d-915a-04137b452a5d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:23:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-833d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 18f7b2d8-85c4-41b2-b5ca-ac089bf973b0 kind: Event lastTimestamp: "2026-05-25T12:23:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:23:46Z" name: success-200-isvc-833d6-predictor.18b2ce72fd44a3b3 namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: a3a5e656-8d16-40a3-9750-2393d875c6b0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:24:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-833d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 18f7b2d8-85c4-41b2-b5ca-ac089bf973b0 kind: Event lastTimestamp: "2026-05-25T12:24:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:24:16Z" name: success-200-isvc-833d6-predictor.18b2ce7d7ac30ef7 namespace: kserve-ci-e2e-test resourceVersion: "15749" uid: ef70733c-2324-416b-b94d-23b3f19106d1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:24:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-833d6-predictor namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 18f7b2d8-85c4-41b2-b5ca-ac089bf973b0 kind: Event lastTimestamp: "2026-05-25T12:24:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:24:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:24:16Z" name: success-200-isvc-833d6-predictor.18b2ce7d7b4f114b namespace: kserve-ci-e2e-test resourceVersion: "15750" uid: 018aaa64-abb7-48a5-9daf-f817ba68be91 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:23:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-833d6 namespace: kserve-ci-e2e-test resourceVersion: "14891" uid: 76ecfbf7-3e55-4534-810c-48449c524bf4 kind: Event lastTimestamp: "2026-05-25T12:23:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:23:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:23:49Z" name: success-200-isvc-833d6.18b2ce6f7dbf078d namespace: kserve-ci-e2e-test resourceVersion: "15495" uid: 290a05be-7886-4656-a7ce-da6a8110b521 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-833d6 namespace: kserve-ci-e2e-test resourceVersion: "15492" uid: 76ecfbf7-3e55-4534-810c-48449c524bf4 kind: Event lastTimestamp: "2026-05-25T12:23:49Z" message: InferenceService [success-200-isvc-833d6] is Ready metadata: creationTimestamp: "2026-05-25T12:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:23:49Z" name: success-200-isvc-833d6.18b2ce7ab4546dd3 namespace: kserve-ci-e2e-test resourceVersion: "15494" uid: 4c8aa765-2827-4c5e-b04e-5ed6fe7d7d12 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:43:31.573772Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr namespace: kserve-ci-e2e-test resourceVersion: "25108" uid: 7a6774c9-5000-4f2c-9ece-20506dde280f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8902c-predictor-b6f848b6b-rmjnr to ip-10-0-142-53.ec2.internal metadata: creationTimestamp: "2026-05-25T12:43:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:43:31Z" name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr.18b2cf8e00feed64 namespace: kserve-ci-e2e-test resourceVersion: "25116" uid: 20550661-f2b6-4a63-b58d-babed6370770 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr namespace: kserve-ci-e2e-test resourceVersion: "25110" uid: 7a6774c9-5000-4f2c-9ece-20506dde280f kind: Event lastTimestamp: "2026-05-25T12:43:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-8902c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:31Z" name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr.18b2cf8e0d33ede0 namespace: kserve-ci-e2e-test resourceVersion: "25150" uid: 3ad89c2d-4a6e-4b20-a05b-50671a00f07a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr namespace: kserve-ci-e2e-test resourceVersion: "25119" uid: 7a6774c9-5000-4f2c-9ece-20506dde280f kind: Event lastTimestamp: "2026-05-25T12:43:32Z" message: Add eth0 [10.134.0.52/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:43:32Z" name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr.18b2cf8e3f454752 namespace: kserve-ci-e2e-test resourceVersion: "25169" uid: c884f352-cbcc-4974-a394-fdd168a39caa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr namespace: kserve-ci-e2e-test resourceVersion: "25110" uid: 7a6774c9-5000-4f2c-9ece-20506dde280f kind: Event lastTimestamp: "2026-05-25T12:43:32Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T12:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:32Z" name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr.18b2cf8e40dd1962 namespace: kserve-ci-e2e-test resourceVersion: "25171" uid: 1053ddd5-918a-4fe0-bf13-d7abdda38bc5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr namespace: kserve-ci-e2e-test resourceVersion: "25110" uid: 7a6774c9-5000-4f2c-9ece-20506dde280f kind: Event lastTimestamp: "2026-05-25T12:43:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:32Z" name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr.18b2cf8e455006ab namespace: kserve-ci-e2e-test resourceVersion: "25172" uid: 42a2bb71-22e5-4229-a0d9-cf3803c99c40 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr namespace: kserve-ci-e2e-test resourceVersion: "25110" uid: 7a6774c9-5000-4f2c-9ece-20506dde280f kind: Event lastTimestamp: "2026-05-25T12:43:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:32Z" name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr.18b2cf8e45f0b751 namespace: kserve-ci-e2e-test resourceVersion: "25173" uid: ad60592f-b797-4273-a626-8b776e6ce728 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr namespace: kserve-ci-e2e-test resourceVersion: "25110" uid: 7a6774c9-5000-4f2c-9ece-20506dde280f kind: Event lastTimestamp: "2026-05-25T12:43:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T12:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:32Z" name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr.18b2cf8e45fff673 namespace: kserve-ci-e2e-test resourceVersion: "25174" uid: 4e78c7b1-1d76-4334-9195-306145f3a40e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr namespace: kserve-ci-e2e-test resourceVersion: "25110" uid: 7a6774c9-5000-4f2c-9ece-20506dde280f kind: Event lastTimestamp: "2026-05-25T12:43:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:32Z" name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr.18b2cf8e4ad7d45b namespace: kserve-ci-e2e-test resourceVersion: "25179" uid: abc714af-925c-4c7d-aa2b-cddd71cf7106 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr namespace: kserve-ci-e2e-test resourceVersion: "25110" uid: 7a6774c9-5000-4f2c-9ece-20506dde280f kind: Event lastTimestamp: "2026-05-25T12:43:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:43:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:32Z" name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr.18b2cf8e4b50939c namespace: kserve-ci-e2e-test resourceVersion: "25180" uid: d166559d-0a8c-4883-9c88-10d38d6d0803 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T12:43:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr namespace: kserve-ci-e2e-test resourceVersion: "25110" uid: 7a6774c9-5000-4f2c-9ece-20506dde280f kind: Event lastTimestamp: "2026-05-25T12:44:10Z" message: 'Readiness probe failed: dial tcp 10.134.0.52:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:43:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:44:10Z" name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr.18b2cf8e7f3d2121 namespace: kserve-ci-e2e-test resourceVersion: "25566" uid: 1510b50b-0ff3-4bbc-989a-bef5c46ab498 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:52:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr namespace: kserve-ci-e2e-test resourceVersion: "25110" uid: 7a6774c9-5000-4f2c-9ece-20506dde280f kind: Event lastTimestamp: "2026-05-25T12:52:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:52:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:52:46Z" name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr.18b2d00f395bd2ff namespace: kserve-ci-e2e-test resourceVersion: "29522" uid: 3f79e51b-6250-4ce5-8ea9-33f264c875ae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:52:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr namespace: kserve-ci-e2e-test resourceVersion: "25110" uid: 7a6774c9-5000-4f2c-9ece-20506dde280f kind: Event lastTimestamp: "2026-05-25T12:52:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:52:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:52:46Z" name: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr.18b2d00f395c88a4 namespace: kserve-ci-e2e-test resourceVersion: "29524" uid: 1beec876-21b9-496d-8a5b-fce0b4ead6b6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8902c-predictor-b6f848b6b namespace: kserve-ci-e2e-test resourceVersion: "25092" uid: 417a43be-833c-453f-9732-ffe763f12a31 kind: Event lastTimestamp: "2026-05-25T12:43:31Z" message: 'Created pod: success-200-isvc-8902c-predictor-b6f848b6b-rmjnr' metadata: creationTimestamp: "2026-05-25T12:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:43:31Z" name: success-200-isvc-8902c-predictor-b6f848b6b.18b2cf8e0086b711 namespace: kserve-ci-e2e-test resourceVersion: "25111" uid: 22c30143-7d85-4907-a957-492638409c03 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8902c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25083" uid: 402e59c3-8ab1-41be-ba6e-ae9e7bd1d22f kind: Event lastTimestamp: "2026-05-25T12:43:31Z" message: Scaled up replica set success-200-isvc-8902c-predictor-b6f848b6b from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:43:31Z" name: success-200-isvc-8902c-predictor.18b2cf8dfe389953 namespace: kserve-ci-e2e-test resourceVersion: "25095" uid: f07a4b6b-0d8a-42e2-8b34-531a43e167d0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:43:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8902c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25088" uid: aab6b023-298b-40ee-913a-c47c3cb8e60c kind: Event lastTimestamp: "2026-05-25T12:44:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:44:01Z" name: success-200-isvc-8902c-predictor.18b2cf917d27b40a namespace: kserve-ci-e2e-test resourceVersion: "25476" uid: f9299117-17ff-430f-91bc-e8499daf618a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:43:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8902c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25088" uid: aab6b023-298b-40ee-913a-c47c3cb8e60c kind: Event lastTimestamp: "2026-05-25T12:44:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:43:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:44:01Z" name: success-200-isvc-8902c-predictor.18b2cf917d95cf08 namespace: kserve-ci-e2e-test resourceVersion: "25478" uid: 5c33a483-e0e4-41c8-a089-b97160cdb3f6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8902c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25322" uid: aab6b023-298b-40ee-913a-c47c3cb8e60c kind: Event lastTimestamp: "2026-05-25T12:44:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:44:31Z" name: success-200-isvc-8902c-predictor.18b2cf987b0d7811 namespace: kserve-ci-e2e-test resourceVersion: "25751" uid: 2d0a82ec-e9a6-4138-9db4-406bb6d298d5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8902c-predictor namespace: kserve-ci-e2e-test resourceVersion: "25322" uid: aab6b023-298b-40ee-913a-c47c3cb8e60c kind: Event lastTimestamp: "2026-05-25T12:44:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:44:31Z" name: success-200-isvc-8902c-predictor.18b2cf987b7da315 namespace: kserve-ci-e2e-test resourceVersion: "25752" uid: 33e8fb1d-58d7-4b43-8338-7dad5531c0e6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:43:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8902c namespace: kserve-ci-e2e-test resourceVersion: "25064" uid: 6f3ce20f-e54c-4d80-852c-8bb60b7befd4 kind: Event lastTimestamp: "2026-05-25T12:44:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:44:20Z" name: success-200-isvc-8902c.18b2cf8dfe09e340 namespace: kserve-ci-e2e-test resourceVersion: "25661" uid: 3483242e-255a-444d-b024-916a1135522a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8902c namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: 6f3ce20f-e54c-4d80-852c-8bb60b7befd4 kind: Event lastTimestamp: "2026-05-25T12:44:20Z" message: InferenceService [success-200-isvc-8902c] is Ready metadata: creationTimestamp: "2026-05-25T12:44:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:44:20Z" name: success-200-isvc-8902c.18b2cf9977644e45 namespace: kserve-ci-e2e-test resourceVersion: "25657" uid: 620d745b-29eb-4b8c-be13-8a5b0e38131d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:32:16.022015Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm namespace: kserve-ci-e2e-test resourceVersion: "19236" uid: f2f7c261-0a24-4aa1-b841-a92ba3120164 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm to ip-10-0-142-53.ec2.internal metadata: creationTimestamp: "2026-05-25T12:32:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:32:16Z" name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm.18b2cef0b6f9218b namespace: kserve-ci-e2e-test resourceVersion: "19241" uid: 82109d8a-6d81-42d7-83b9-227d10e23c2a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm namespace: kserve-ci-e2e-test resourceVersion: "19238" uid: f2f7c261-0a24-4aa1-b841-a92ba3120164 kind: Event lastTimestamp: "2026-05-25T12:32:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-acbca-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:16Z" name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm.18b2cef0bf7978b1 namespace: kserve-ci-e2e-test resourceVersion: "19280" uid: 1d4fd0d5-7ad2-489f-b0f8-7d006ded81b3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm namespace: kserve-ci-e2e-test resourceVersion: "19254" uid: f2f7c261-0a24-4aa1-b841-a92ba3120164 kind: Event lastTimestamp: "2026-05-25T12:32:17Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:32:17Z" name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm.18b2cef0f4d84fc5 namespace: kserve-ci-e2e-test resourceVersion: "19307" uid: e5bb638e-ead3-4065-95b4-a46302fd9dbb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm namespace: kserve-ci-e2e-test resourceVersion: "19238" uid: f2f7c261-0a24-4aa1-b841-a92ba3120164 kind: Event lastTimestamp: "2026-05-25T12:32:17Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T12:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:17Z" name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm.18b2cef0f7cd770e namespace: kserve-ci-e2e-test resourceVersion: "19309" uid: c0453e47-a572-4d51-95f5-cae7d236443f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm namespace: kserve-ci-e2e-test resourceVersion: "19238" uid: f2f7c261-0a24-4aa1-b841-a92ba3120164 kind: Event lastTimestamp: "2026-05-25T12:32:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:17Z" name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm.18b2cef0fb873c8a namespace: kserve-ci-e2e-test resourceVersion: "19313" uid: 628c965b-99e5-4f22-9fb0-25cbea41f3f7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm namespace: kserve-ci-e2e-test resourceVersion: "19238" uid: f2f7c261-0a24-4aa1-b841-a92ba3120164 kind: Event lastTimestamp: "2026-05-25T12:32:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:17Z" name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm.18b2cef0fbfb5865 namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: 38a8133f-af88-4e2e-8a07-ffebb5955009 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm namespace: kserve-ci-e2e-test resourceVersion: "19238" uid: f2f7c261-0a24-4aa1-b841-a92ba3120164 kind: Event lastTimestamp: "2026-05-25T12:32:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T12:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:17Z" name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm.18b2cef0fc06d61a namespace: kserve-ci-e2e-test resourceVersion: "19315" uid: b347c0b6-73e5-4030-befb-433aeb8a4175 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm namespace: kserve-ci-e2e-test resourceVersion: "19238" uid: f2f7c261-0a24-4aa1-b841-a92ba3120164 kind: Event lastTimestamp: "2026-05-25T12:32:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:17Z" name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm.18b2cef0ff95a0ed namespace: kserve-ci-e2e-test resourceVersion: "19317" uid: f1f19982-828b-4164-8e15-bc3d7687ec2f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm namespace: kserve-ci-e2e-test resourceVersion: "19238" uid: f2f7c261-0a24-4aa1-b841-a92ba3120164 kind: Event lastTimestamp: "2026-05-25T12:32:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:17Z" name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm.18b2cef100466e3d namespace: kserve-ci-e2e-test resourceVersion: "19318" uid: c6ade254-3f9a-424f-b320-49f217c82886 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-25T12:32:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm namespace: kserve-ci-e2e-test resourceVersion: "19238" uid: f2f7c261-0a24-4aa1-b841-a92ba3120164 kind: Event lastTimestamp: "2026-05-25T12:33:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:04Z" name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm.18b2cef1319a71df namespace: kserve-ci-e2e-test resourceVersion: "19924" uid: fba01694-58fe-44cb-b90a-8b3f74251cd1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm namespace: kserve-ci-e2e-test resourceVersion: "19238" uid: f2f7c261-0a24-4aa1-b841-a92ba3120164 kind: Event lastTimestamp: "2026-05-25T12:33:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:36Z" name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm.18b2cf0366e288ef namespace: kserve-ci-e2e-test resourceVersion: "20306" uid: 314c8752-1b5e-4522-b029-034a516e45ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm namespace: kserve-ci-e2e-test resourceVersion: "19238" uid: f2f7c261-0a24-4aa1-b841-a92ba3120164 kind: Event lastTimestamp: "2026-05-25T12:33:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:36Z" name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm.18b2cf0366e3f574 namespace: kserve-ci-e2e-test resourceVersion: "20311" uid: 1ece0014-ae10-4cc4-bd68-f7d62e6b5aa4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm namespace: kserve-ci-e2e-test resourceVersion: "19238" uid: f2f7c261-0a24-4aa1-b841-a92ba3120164 kind: Event lastTimestamp: "2026-05-25T12:33:39Z" message: 'Readiness probe failed: Get "https://10.134.0.42:8643/healthz": dial tcp 10.134.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:33:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:33:39Z" name: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm.18b2cf040dcbc5ec namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: b69afe99-bed1-4781-be61-21ac376f96f4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-acbca-predictor-59cffd4b6b namespace: kserve-ci-e2e-test resourceVersion: "19221" uid: 1afd4451-615d-4d93-b8a1-54bc74c68af0 kind: Event lastTimestamp: "2026-05-25T12:32:16Z" message: 'Created pod: success-200-isvc-acbca-predictor-59cffd4b6b-zwcgm' metadata: creationTimestamp: "2026-05-25T12:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:32:16Z" name: success-200-isvc-acbca-predictor-59cffd4b6b.18b2cef0b65d7cf8 namespace: kserve-ci-e2e-test resourceVersion: "19237" uid: b63c6b4b-257d-4409-bdba-159454f76f49 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-acbca-predictor namespace: kserve-ci-e2e-test resourceVersion: "19216" uid: 1e4d2a22-246e-4596-808f-aea39faf47f2 kind: Event lastTimestamp: "2026-05-25T12:32:15Z" message: Scaled up replica set success-200-isvc-acbca-predictor-59cffd4b6b from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:32:15Z" name: success-200-isvc-acbca-predictor.18b2cef0b31dbdef namespace: kserve-ci-e2e-test resourceVersion: "19224" uid: 69a44ad6-0c55-4488-ba61-4e480f096a27 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:32:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-acbca-predictor namespace: kserve-ci-e2e-test resourceVersion: "19220" uid: a018e91a-7e02-4515-8b97-46a28fdd13c6 kind: Event lastTimestamp: "2026-05-25T12:33:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:32:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:33:01Z" name: success-200-isvc-acbca-predictor.18b2cef4324890e0 namespace: kserve-ci-e2e-test resourceVersion: "19895" uid: 45fe9593-3f32-4868-82e0-a4ada9df562b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:32:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-acbca-predictor namespace: kserve-ci-e2e-test resourceVersion: "19220" uid: a018e91a-7e02-4515-8b97-46a28fdd13c6 kind: Event lastTimestamp: "2026-05-25T12:33:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:32:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:33:01Z" name: success-200-isvc-acbca-predictor.18b2cef432f7c3d3 namespace: kserve-ci-e2e-test resourceVersion: "19896" uid: 381b472c-187d-47b5-897d-0c47ac81ce84 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:32:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-acbca-predictor namespace: kserve-ci-e2e-test resourceVersion: "19449" uid: a018e91a-7e02-4515-8b97-46a28fdd13c6 kind: Event lastTimestamp: "2026-05-25T12:33:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:32:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:33:31Z" name: success-200-isvc-acbca-predictor.18b2cef7b2a62e7a namespace: kserve-ci-e2e-test resourceVersion: "20193" uid: 8b53ca77-c0a0-4b0b-a295-51fed00cca05 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:32:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-acbca-predictor namespace: kserve-ci-e2e-test resourceVersion: "19449" uid: a018e91a-7e02-4515-8b97-46a28fdd13c6 kind: Event lastTimestamp: "2026-05-25T12:33:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:32:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:33:31Z" name: success-200-isvc-acbca-predictor.18b2cef7b36aa823 namespace: kserve-ci-e2e-test resourceVersion: "20194" uid: a3df9e6d-d96d-475b-b48a-2fdda67b7afc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:32:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-acbca namespace: kserve-ci-e2e-test resourceVersion: "19197" uid: 3b7a2e53-cb15-4a42-b06a-e9621e5b516c kind: Event lastTimestamp: "2026-05-25T12:33:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:33:14Z" name: success-200-isvc-acbca.18b2cef0b2faa3ce namespace: kserve-ci-e2e-test resourceVersion: "20027" uid: bf3906b0-2baa-4f35-a674-adc6fdc75039 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-acbca namespace: kserve-ci-e2e-test resourceVersion: "19197" uid: 3b7a2e53-cb15-4a42-b06a-e9621e5b516c kind: Event lastTimestamp: "2026-05-25T12:32:15Z" message: 'Failed to update status for InferenceService "success-200-isvc-acbca": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-acbca": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:32:15Z" name: success-200-isvc-acbca.18b2cef0b39f3e50 namespace: kserve-ci-e2e-test resourceVersion: "19230" uid: 23156821-bf34-4f77-b372-35fefc29a0e7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-acbca namespace: kserve-ci-e2e-test resourceVersion: "19197" uid: 3b7a2e53-cb15-4a42-b06a-e9621e5b516c kind: Event lastTimestamp: "2026-05-25T12:32:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-acbca": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:32:15Z" name: success-200-isvc-acbca.18b2cef0b39f8be8 namespace: kserve-ci-e2e-test resourceVersion: "19233" uid: e22ec5e0-4d39-4c6e-814e-0f5c6607f43b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:33:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-acbca namespace: kserve-ci-e2e-test resourceVersion: "20025" uid: 3b7a2e53-cb15-4a42-b06a-e9621e5b516c kind: Event lastTimestamp: "2026-05-25T12:33:14Z" message: InferenceService [success-200-isvc-acbca] is Ready metadata: creationTimestamp: "2026-05-25T12:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:33:14Z" name: success-200-isvc-acbca.18b2cefe42888649 namespace: kserve-ci-e2e-test resourceVersion: "20026" uid: a4dac1aa-9cba-4e55-9c0a-673ce32ad76c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:31:46.015874Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx namespace: kserve-ci-e2e-test resourceVersion: "18876" uid: 81f25695-1e0c-4e1c-beda-e1d250bf364b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-bcff4-predictor-66c9fb5576-scwgx to ip-10-0-142-53.ec2.internal metadata: creationTimestamp: "2026-05-25T12:31:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:31:46Z" name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx.18b2cee9ba77c558 namespace: kserve-ci-e2e-test resourceVersion: "18887" uid: 76b42d98-59ed-4417-ac0a-92b01527e3ef reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx namespace: kserve-ci-e2e-test resourceVersion: "18900" uid: 81f25695-1e0c-4e1c-beda-e1d250bf364b kind: Event lastTimestamp: "2026-05-25T12:31:46Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:31:46Z" name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx.18b2cee9d54aa8cf namespace: kserve-ci-e2e-test resourceVersion: "18920" uid: 0f0cf788-d6dd-4d93-92a8-ad0a99366f66 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx namespace: kserve-ci-e2e-test resourceVersion: "18881" uid: 81f25695-1e0c-4e1c-beda-e1d250bf364b kind: Event lastTimestamp: "2026-05-25T12:31:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T12:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:31:46Z" name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx.18b2cee9d7108edb namespace: kserve-ci-e2e-test resourceVersion: "18924" uid: 00ee29d1-3625-4225-8929-e578fe4ab30d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx namespace: kserve-ci-e2e-test resourceVersion: "18881" uid: 81f25695-1e0c-4e1c-beda-e1d250bf364b kind: Event lastTimestamp: "2026-05-25T12:31:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:31:46Z" name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx.18b2cee9d9dfb72e namespace: kserve-ci-e2e-test resourceVersion: "18925" uid: 9f330eb1-cc21-4c70-9979-30d5e09a7f09 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx namespace: kserve-ci-e2e-test resourceVersion: "18881" uid: 81f25695-1e0c-4e1c-beda-e1d250bf364b kind: Event lastTimestamp: "2026-05-25T12:31:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:31:46Z" name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx.18b2cee9da506094 namespace: kserve-ci-e2e-test resourceVersion: "18926" uid: 60f72254-672e-4934-8f16-0dc403f10959 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx namespace: kserve-ci-e2e-test resourceVersion: "18881" uid: 81f25695-1e0c-4e1c-beda-e1d250bf364b kind: Event lastTimestamp: "2026-05-25T12:31:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T12:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:31:46Z" name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx.18b2cee9da5c5e44 namespace: kserve-ci-e2e-test resourceVersion: "18927" uid: 731c77b4-33c8-461c-8fef-e5d3ab4ef41d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx namespace: kserve-ci-e2e-test resourceVersion: "18881" uid: 81f25695-1e0c-4e1c-beda-e1d250bf364b kind: Event lastTimestamp: "2026-05-25T12:31:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:31:46Z" name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx.18b2cee9ddef7a02 namespace: kserve-ci-e2e-test resourceVersion: "18929" uid: abd76916-1000-4112-a6b6-2157f1c16b9c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx namespace: kserve-ci-e2e-test resourceVersion: "18881" uid: 81f25695-1e0c-4e1c-beda-e1d250bf364b kind: Event lastTimestamp: "2026-05-25T12:31:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:31:46Z" name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx.18b2cee9de76ea22 namespace: kserve-ci-e2e-test resourceVersion: "18930" uid: a8f74857-8695-4e60-8cbc-8c897065f4f3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:31:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx namespace: kserve-ci-e2e-test resourceVersion: "18881" uid: 81f25695-1e0c-4e1c-beda-e1d250bf364b kind: Event lastTimestamp: "2026-05-25T12:32:23Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:31:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:23Z" name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx.18b2ceea2e0c349e namespace: kserve-ci-e2e-test resourceVersion: "19397" uid: a7f2df55-39f4-4451-8cfb-f84ae64c60a4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx namespace: kserve-ci-e2e-test resourceVersion: "18881" uid: 81f25695-1e0c-4e1c-beda-e1d250bf364b kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:56Z" name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx.18b2cefa1d27b8eb namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: ee4b6e65-9ba8-4691-aa87-21eff77737d1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx namespace: kserve-ci-e2e-test resourceVersion: "18881" uid: 81f25695-1e0c-4e1c-beda-e1d250bf364b kind: Event lastTimestamp: "2026-05-25T12:32:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:56Z" name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx.18b2cefa1d28215d namespace: kserve-ci-e2e-test resourceVersion: "19780" uid: 5139d5f8-cf8b-4bfa-a50a-f4d9ca8ee8d9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx namespace: kserve-ci-e2e-test resourceVersion: "18881" uid: 81f25695-1e0c-4e1c-beda-e1d250bf364b kind: Event lastTimestamp: "2026-05-25T12:32:58Z" message: 'Readiness probe failed: Get "https://10.134.0.40:8643/healthz": dial tcp 10.134.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:32:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:58Z" name: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx.18b2cefab621f423 namespace: kserve-ci-e2e-test resourceVersion: "19870" uid: 9aa4c228-9cc8-4cbd-aed7-0750c8d8d0c8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-bcff4-predictor-66c9fb5576 namespace: kserve-ci-e2e-test resourceVersion: "18850" uid: 6f8ec730-7ad8-49c0-858f-fa827568a440 kind: Event lastTimestamp: "2026-05-25T12:31:46Z" message: 'Created pod: success-200-isvc-bcff4-predictor-66c9fb5576-scwgx' metadata: creationTimestamp: "2026-05-25T12:31:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:31:46Z" name: success-200-isvc-bcff4-predictor-66c9fb5576.18b2cee9b9de0abb namespace: kserve-ci-e2e-test resourceVersion: "18880" uid: f6b90021-8d73-4ee0-93c3-e06ad8137527 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-bcff4-predictor namespace: kserve-ci-e2e-test resourceVersion: "18847" uid: 39093d05-6b01-435d-9fa6-b80340286796 kind: Event lastTimestamp: "2026-05-25T12:31:45Z" message: Scaled up replica set success-200-isvc-bcff4-predictor-66c9fb5576 from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:31:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:31:45Z" name: success-200-isvc-bcff4-predictor.18b2cee9b4ca7f32 namespace: kserve-ci-e2e-test resourceVersion: "18851" uid: 900a1de9-fedf-46c6-8741-066e797c90ed reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:32:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bcff4-predictor namespace: kserve-ci-e2e-test resourceVersion: "18849" uid: f29534ac-a5fd-4fee-9b5b-3c34d6bb8ec1 kind: Event lastTimestamp: "2026-05-25T12:32:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:32:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:32:30Z" name: success-200-isvc-bcff4-predictor.18b2ceed33cc1391 namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: 4276c0f4-9f99-4d53-9c84-53ddbf959a4b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:32:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bcff4-predictor namespace: kserve-ci-e2e-test resourceVersion: "18849" uid: f29534ac-a5fd-4fee-9b5b-3c34d6bb8ec1 kind: Event lastTimestamp: "2026-05-25T12:32:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:32:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:32:31Z" name: success-200-isvc-bcff4-predictor.18b2ceed344d6713 namespace: kserve-ci-e2e-test resourceVersion: "19454" uid: 649364d8-3c64-481d-928e-967341b07472 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:32:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bcff4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19066" uid: f29534ac-a5fd-4fee-9b5b-3c34d6bb8ec1 kind: Event lastTimestamp: "2026-05-25T12:32:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:32:46Z" name: success-200-isvc-bcff4-predictor.18b2cef0b3286008 namespace: kserve-ci-e2e-test resourceVersion: "19601" uid: 614298b7-d3ff-4157-8b4f-1a0b286eabd5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:32:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bcff4-predictor namespace: kserve-ci-e2e-test resourceVersion: "19066" uid: f29534ac-a5fd-4fee-9b5b-3c34d6bb8ec1 kind: Event lastTimestamp: "2026-05-25T12:32:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:32:46Z" name: success-200-isvc-bcff4-predictor.18b2cef0b3d15873 namespace: kserve-ci-e2e-test resourceVersion: "19607" uid: c74df126-fd3b-47d4-a18f-4eaca72be4b3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T12:31:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bcff4 namespace: kserve-ci-e2e-test resourceVersion: "18828" uid: eee79151-8eaa-449b-aea9-19e457dc31bf kind: Event lastTimestamp: "2026-05-25T12:32:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:31:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:32:34Z" name: success-200-isvc-bcff4.18b2cee9b4cd4930 namespace: kserve-ci-e2e-test resourceVersion: "19493" uid: 763e6cd2-5376-47e4-94e7-776c333dcdfe reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bcff4 namespace: kserve-ci-e2e-test resourceVersion: "18828" uid: eee79151-8eaa-449b-aea9-19e457dc31bf kind: Event lastTimestamp: "2026-05-25T12:31:45Z" message: 'Failed to update status for InferenceService "success-200-isvc-bcff4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-bcff4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:31:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:31:45Z" name: success-200-isvc-bcff4.18b2cee9b55eead1 namespace: kserve-ci-e2e-test resourceVersion: "18856" uid: 8faccc71-6567-4d1f-b7f1-c588cb56cedb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bcff4 namespace: kserve-ci-e2e-test resourceVersion: "18828" uid: eee79151-8eaa-449b-aea9-19e457dc31bf kind: Event lastTimestamp: "2026-05-25T12:31:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-bcff4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:31:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:31:45Z" name: success-200-isvc-bcff4.18b2cee9b55f5990 namespace: kserve-ci-e2e-test resourceVersion: "18857" uid: 28b461cc-74d7-49e8-9369-2998d6f33fea reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:32:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bcff4 namespace: kserve-ci-e2e-test resourceVersion: "19487" uid: eee79151-8eaa-449b-aea9-19e457dc31bf kind: Event lastTimestamp: "2026-05-25T12:32:34Z" message: InferenceService [success-200-isvc-bcff4] is Ready metadata: creationTimestamp: "2026-05-25T12:32:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:32:34Z" name: success-200-isvc-bcff4.18b2cef4eaa450d6 namespace: kserve-ci-e2e-test resourceVersion: "19489" uid: 4d8b0f52-dd6d-4093-b42b-3ae6b58b2ccd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:42:21.262201Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv namespace: kserve-ci-e2e-test resourceVersion: "24214" uid: 90636a58-53a5-4c10-864f-cdf6647bd677 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv to ip-10-0-142-53.ec2.internal metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:42:21Z" name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv.18b2cf7da2197357 namespace: kserve-ci-e2e-test resourceVersion: "24223" uid: 2d6a34ab-c06a-425c-b72c-1aec8399579b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv namespace: kserve-ci-e2e-test resourceVersion: "24219" uid: 90636a58-53a5-4c10-864f-cdf6647bd677 kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-fd4b4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:21Z" name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv.18b2cf7db247d4f8 namespace: kserve-ci-e2e-test resourceVersion: "24258" uid: ba17e06b-8cf6-4e26-9a4f-089eb5dc1883 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv namespace: kserve-ci-e2e-test resourceVersion: "24231" uid: 90636a58-53a5-4c10-864f-cdf6647bd677 kind: Event lastTimestamp: "2026-05-25T12:42:22Z" message: Add eth0 [10.134.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:42:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:42:22Z" name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv.18b2cf7de087a514 namespace: kserve-ci-e2e-test resourceVersion: "24279" uid: 64394bf6-6da2-46ed-a55b-3e23194b5423 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv namespace: kserve-ci-e2e-test resourceVersion: "24219" uid: 90636a58-53a5-4c10-864f-cdf6647bd677 kind: Event lastTimestamp: "2026-05-25T12:42:22Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-25T12:42:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:22Z" name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv.18b2cf7de1f21883 namespace: kserve-ci-e2e-test resourceVersion: "24281" uid: 5116b378-6398-4d58-8684-8ca1d44b00c6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv namespace: kserve-ci-e2e-test resourceVersion: "24219" uid: 90636a58-53a5-4c10-864f-cdf6647bd677 kind: Event lastTimestamp: "2026-05-25T12:42:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-25T12:42:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:22Z" name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv.18b2cf7de5f7c8cd namespace: kserve-ci-e2e-test resourceVersion: "24287" uid: 8ad59145-7f1d-47a9-96aa-0c50e7f6f63a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv namespace: kserve-ci-e2e-test resourceVersion: "24219" uid: 90636a58-53a5-4c10-864f-cdf6647bd677 kind: Event lastTimestamp: "2026-05-25T12:42:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-25T12:42:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:22Z" name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv.18b2cf7de687605e namespace: kserve-ci-e2e-test resourceVersion: "24288" uid: a0287f8a-563c-4102-950f-04306df2e9e0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv namespace: kserve-ci-e2e-test resourceVersion: "24219" uid: 90636a58-53a5-4c10-864f-cdf6647bd677 kind: Event lastTimestamp: "2026-05-25T12:42:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-25T12:42:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:22Z" name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv.18b2cf7de694e355 namespace: kserve-ci-e2e-test resourceVersion: "24289" uid: def687e3-296d-4531-a75a-129b2797d6d2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv namespace: kserve-ci-e2e-test resourceVersion: "24219" uid: 90636a58-53a5-4c10-864f-cdf6647bd677 kind: Event lastTimestamp: "2026-05-25T12:42:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-25T12:42:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:22Z" name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv.18b2cf7de9aca2bf namespace: kserve-ci-e2e-test resourceVersion: "24290" uid: a10d8ece-9836-4587-a10c-53804789e531 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv namespace: kserve-ci-e2e-test resourceVersion: "24219" uid: 90636a58-53a5-4c10-864f-cdf6647bd677 kind: Event lastTimestamp: "2026-05-25T12:42:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:42:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:22Z" name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv.18b2cf7dea26276e namespace: kserve-ci-e2e-test resourceVersion: "24291" uid: 8dfd4b4f-020d-4025-9729-467352dedc39 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:42:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv namespace: kserve-ci-e2e-test resourceVersion: "24219" uid: 90636a58-53a5-4c10-864f-cdf6647bd677 kind: Event lastTimestamp: "2026-05-25T12:42:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:42:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:42:59Z" name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv.18b2cf7e20c5ecd6 namespace: kserve-ci-e2e-test resourceVersion: "24728" uid: deb42915-d047-4d67-b701-9c01dbe4881a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv namespace: kserve-ci-e2e-test resourceVersion: "24219" uid: 90636a58-53a5-4c10-864f-cdf6647bd677 kind: Event lastTimestamp: "2026-05-25T12:43:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-25T12:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:31Z" name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv.18b2cf8e004bc3cf namespace: kserve-ci-e2e-test resourceVersion: "25109" uid: 37e4f3a5-c8a0-44be-86bb-82e6bcdf89b7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv namespace: kserve-ci-e2e-test resourceVersion: "24219" uid: 90636a58-53a5-4c10-864f-cdf6647bd677 kind: Event lastTimestamp: "2026-05-25T12:43:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-25T12:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:31Z" name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv.18b2cf8e00516536 namespace: kserve-ci-e2e-test resourceVersion: "25112" uid: 5922a3cb-4375-4897-ac41-0130cbc8c803 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv namespace: kserve-ci-e2e-test resourceVersion: "24219" uid: 90636a58-53a5-4c10-864f-cdf6647bd677 kind: Event lastTimestamp: "2026-05-25T12:43:34Z" message: 'Readiness probe failed: Get "https://10.134.0.48:8643/healthz": dial tcp 10.134.0.48:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-25T12:43:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:43:34Z" name: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv.18b2cf8ea8f786a6 namespace: kserve-ci-e2e-test resourceVersion: "25207" uid: f110f4be-081e-47d0-9d67-cc41d6d235d7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-53.ec2.internal source: component: kubelet host: ip-10-0-142-53.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-fd4b4-predictor-59d7bccbf8 namespace: kserve-ci-e2e-test resourceVersion: "24193" uid: 5b16cfc9-fc64-4a4d-acf3-e21f95803f44 kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: 'Created pod: success-200-isvc-fd4b4-predictor-59d7bccbf8-pl6qv' metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:42:21Z" name: success-200-isvc-fd4b4-predictor-59d7bccbf8.18b2cf7da1a70264 namespace: kserve-ci-e2e-test resourceVersion: "24218" uid: 27aea2b0-17c2-42a1-a7a1-f3d12064b28e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-fd4b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "24191" uid: d9943cd3-367d-454f-a7e8-6aab05cebb49 kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: Scaled up replica set success-200-isvc-fd4b4-predictor-59d7bccbf8 from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:42:21Z" name: success-200-isvc-fd4b4-predictor.18b2cf7d9e312b22 namespace: kserve-ci-e2e-test resourceVersion: "24196" uid: d2044dcb-63ce-4f87-b840-5c1227784a68 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:42:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fd4b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "24192" uid: d7f76113-8d61-4ef7-bf47-2b6d86aa9639 kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:42:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:42:51Z" name: success-200-isvc-fd4b4-predictor.18b2cf811cf96a8c namespace: kserve-ci-e2e-test resourceVersion: "24600" uid: 1b4c7923-9f68-432a-ad3d-07cea0bd395f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:42:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fd4b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "24192" uid: d7f76113-8d61-4ef7-bf47-2b6d86aa9639 kind: Event lastTimestamp: "2026-05-25T12:42:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:42:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:42:51Z" name: success-200-isvc-fd4b4-predictor.18b2cf811d735665 namespace: kserve-ci-e2e-test resourceVersion: "24604" uid: 44aa236d-ed70-408a-81e4-b8c7796ce4ad reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:43:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fd4b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "24409" uid: d7f76113-8d61-4ef7-bf47-2b6d86aa9639 kind: Event lastTimestamp: "2026-05-25T12:43:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:43:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:43:21Z" name: success-200-isvc-fd4b4-predictor.18b2cf881b53197e namespace: kserve-ci-e2e-test resourceVersion: "24927" uid: c649843b-5b1f-46e8-8143-250e9b403195 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-25T12:43:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fd4b4-predictor namespace: kserve-ci-e2e-test resourceVersion: "24409" uid: d7f76113-8d61-4ef7-bf47-2b6d86aa9639 kind: Event lastTimestamp: "2026-05-25T12:43:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-25T12:43:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:43:21Z" name: success-200-isvc-fd4b4-predictor.18b2cf881bca52b9 namespace: kserve-ci-e2e-test resourceVersion: "24928" uid: c4d256f9-e327-4fde-a9f1-57c42afaa5ad reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fd4b4 namespace: kserve-ci-e2e-test resourceVersion: "24170" uid: 6577b6ee-0415-4e98-ba10-63213789110c kind: Event lastTimestamp: "2026-05-25T12:43:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:43:09Z" name: success-200-isvc-fd4b4.18b2cf7d9e05dee4 namespace: kserve-ci-e2e-test resourceVersion: "24839" uid: 938dae6e-ffbb-4979-b768-f03f7941b4e2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fd4b4 namespace: kserve-ci-e2e-test resourceVersion: "24170" uid: 6577b6ee-0415-4e98-ba10-63213789110c kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: 'Failed to update status for InferenceService "success-200-isvc-fd4b4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fd4b4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:42:21Z" name: success-200-isvc-fd4b4.18b2cf7d9e749734 namespace: kserve-ci-e2e-test resourceVersion: "24199" uid: 7eb893cd-7570-4d2f-806b-23dd89a27089 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:42:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fd4b4 namespace: kserve-ci-e2e-test resourceVersion: "24170" uid: 6577b6ee-0415-4e98-ba10-63213789110c kind: Event lastTimestamp: "2026-05-25T12:42:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fd4b4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-25T12:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:42:21Z" name: success-200-isvc-fd4b4.18b2cf7d9e74e58e namespace: kserve-ci-e2e-test resourceVersion: "24200" uid: 033bc5ce-79dd-40ca-9208-4309e1e39cf3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:43:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fd4b4 namespace: kserve-ci-e2e-test resourceVersion: "24833" uid: 6577b6ee-0415-4e98-ba10-63213789110c kind: Event lastTimestamp: "2026-05-25T12:43:09Z" message: InferenceService [success-200-isvc-fd4b4] is Ready metadata: creationTimestamp: "2026-05-25T12:43:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:43:09Z" name: success-200-isvc-fd4b4.18b2cf88df4f123e namespace: kserve-ci-e2e-test resourceVersion: "24835" uid: 76088418-ffae-4534-80cb-6dcda0439b9e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:44:01.260153Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-09598-5987fb5786-6srjk namespace: kserve-ci-e2e-test resourceVersion: "25457" uid: 35d829e9-8c0c-4d4a-a685-18450e9ba74f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-09598-5987fb5786-6srjk to ip-10-0-143-107.ec2.internal metadata: creationTimestamp: "2026-05-25T12:44:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:44:01Z" name: switch-graph-09598-5987fb5786-6srjk.18b2cf94ea712d6a namespace: kserve-ci-e2e-test resourceVersion: "25462" uid: 42916775-2e5c-48cc-8389-157efe00004a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:01Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-09598-5987fb5786-6srjk namespace: kserve-ci-e2e-test resourceVersion: "25465" uid: 35d829e9-8c0c-4d4a-a685-18450e9ba74f kind: Event lastTimestamp: "2026-05-25T12:44:01Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:44:01Z" name: switch-graph-09598-5987fb5786-6srjk.18b2cf9503d5f4bb namespace: kserve-ci-e2e-test resourceVersion: "25482" uid: 8df98867-a835-482f-98ba-54477aef323e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-09598} kind: Pod name: switch-graph-09598-5987fb5786-6srjk namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: 35d829e9-8c0c-4d4a-a685-18450e9ba74f kind: Event lastTimestamp: "2026-05-25T12:44:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:41f0ca4b45114a1d81ffad66a667eba2baa4df82ad74c2ffc611dc9cdef0d94a" already present on machine metadata: creationTimestamp: "2026-05-25T12:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:44:01Z" name: switch-graph-09598-5987fb5786-6srjk.18b2cf95052c3d4a namespace: kserve-ci-e2e-test resourceVersion: "25485" uid: 5088a1a9-4d2a-4bca-a80d-d8dac00fc661 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-09598} kind: Pod name: switch-graph-09598-5987fb5786-6srjk namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: 35d829e9-8c0c-4d4a-a685-18450e9ba74f kind: Event lastTimestamp: "2026-05-25T12:44:01Z" message: 'Created container: switch-graph-09598' metadata: creationTimestamp: "2026-05-25T12:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:44:01Z" name: switch-graph-09598-5987fb5786-6srjk.18b2cf9507c63255 namespace: kserve-ci-e2e-test resourceVersion: "25486" uid: 8d2f2bd7-554a-4452-ae75-4c135e5e2391 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-09598} kind: Pod name: switch-graph-09598-5987fb5786-6srjk namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: 35d829e9-8c0c-4d4a-a685-18450e9ba74f kind: Event lastTimestamp: "2026-05-25T12:44:01Z" message: Started container switch-graph-09598 metadata: creationTimestamp: "2026-05-25T12:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:44:01Z" name: switch-graph-09598-5987fb5786-6srjk.18b2cf9508344f7e namespace: kserve-ci-e2e-test resourceVersion: "25487" uid: 9c1c9309-321c-4202-abd6-3222d7d4bd0d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T13:00:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-09598} kind: Pod name: switch-graph-09598-5987fb5786-6srjk namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: 35d829e9-8c0c-4d4a-a685-18450e9ba74f kind: Event lastTimestamp: "2026-05-25T13:00:20Z" message: Stopping container switch-graph-09598 metadata: creationTimestamp: "2026-05-25T13:00:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:00:20Z" name: switch-graph-09598-5987fb5786-6srjk.18b2d078f509fc27 namespace: kserve-ci-e2e-test resourceVersion: "32647" uid: c7f333e1-0f14-425e-a776-7824df6f1def reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-25T13:00:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-09598} kind: Pod name: switch-graph-09598-5987fb5786-6srjk namespace: kserve-ci-e2e-test resourceVersion: "25461" uid: 35d829e9-8c0c-4d4a-a685-18450e9ba74f kind: Event lastTimestamp: "2026-05-25T13:00:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T13:00:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T13:00:43Z" name: switch-graph-09598-5987fb5786-6srjk.18b2d07986b09b41 namespace: kserve-ci-e2e-test resourceVersion: "32876" uid: 566c594c-83ef-4b55-8e1d-f23868281748 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-09598-5987fb5786 namespace: kserve-ci-e2e-test resourceVersion: "25454" uid: d6b83182-e84c-4747-8b85-28aec7fc4d85 kind: Event lastTimestamp: "2026-05-25T12:44:01Z" message: 'Created pod: switch-graph-09598-5987fb5786-6srjk' metadata: creationTimestamp: "2026-05-25T12:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:44:01Z" name: switch-graph-09598-5987fb5786.18b2cf94e9fcd9e3 namespace: kserve-ci-e2e-test resourceVersion: "25459" uid: 53ba6f7b-28a4-4a4b-9a86-43c5070b94b6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-09598 namespace: kserve-ci-e2e-test resourceVersion: "25452" uid: f6857adc-699a-4330-a82d-575cd65ce156 kind: Event lastTimestamp: "2026-05-25T12:44:01Z" message: Scaled up replica set switch-graph-09598-5987fb5786 from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:44:01Z" name: switch-graph-09598.18b2cf94e8f8cba6 namespace: kserve-ci-e2e-test resourceVersion: "25455" uid: 0c0efd89-f577-40c8-8224-195c05f81fd8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:44:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-09598 namespace: kserve-ci-e2e-test resourceVersion: "25468" uid: ddd8ce61-d8ac-429f-9720-312843956450 kind: Event lastTimestamp: "2026-05-25T12:44:01Z" message: InferenceGraph [switch-graph-09598] is Ready metadata: creationTimestamp: "2026-05-25T12:44:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:44:01Z" name: switch-graph-09598.18b2cf94eed2814f namespace: kserve-ci-e2e-test resourceVersion: "25470" uid: c064ace2-fa2b-49ad-a3d6-3211c6ff8840 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-09598 namespace: kserve-ci-e2e-test resourceVersion: "25453" uid: e612cd25-2d83-489b-821f-bc10962a3af3 kind: Event lastTimestamp: "2026-05-25T12:44:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:44:46Z" name: switch-graph-09598.18b2cf9867fa2d3f namespace: kserve-ci-e2e-test resourceVersion: "25888" uid: 5e3162cd-d970-437b-8ebf-428c3ab5972f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-25T12:44:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-09598 namespace: kserve-ci-e2e-test resourceVersion: "25453" uid: e612cd25-2d83-489b-821f-bc10962a3af3 kind: Event lastTimestamp: "2026-05-25T12:44:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:44:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:44:46Z" name: switch-graph-09598.18b2cf98686663e4 namespace: kserve-ci-e2e-test resourceVersion: "25889" uid: 62e2ae36-6a33-4d74-b6b8-bb24c0b42488 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:22:20.782549Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1742e-68c468964c-d9m58 namespace: kserve-ci-e2e-test resourceVersion: "14304" uid: cffb6412-1856-40a9-bc55-12b353ed3168 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-1742e-68c468964c-d9m58 to ip-10-0-143-107.ec2.internal metadata: creationTimestamp: "2026-05-25T12:22:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:22:20Z" name: switch-graph-1742e-68c468964c-d9m58.18b2ce661fefb7d5 namespace: kserve-ci-e2e-test resourceVersion: "14309" uid: bdbbfc00-a400-4dd8-82fb-c6ff32e6b7a6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:20Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1742e-68c468964c-d9m58 namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: cffb6412-1856-40a9-bc55-12b353ed3168 kind: Event lastTimestamp: "2026-05-25T12:22:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-1742e-serving-cert" not found' metadata: creationTimestamp: "2026-05-25T12:22:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:20Z" name: switch-graph-1742e-68c468964c-d9m58.18b2ce662bc82e00 namespace: kserve-ci-e2e-test resourceVersion: "14321" uid: 0a574306-94d9-47ff-a313-36f1927e3c62 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:21Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1742e-68c468964c-d9m58 namespace: kserve-ci-e2e-test resourceVersion: "14314" uid: cffb6412-1856-40a9-bc55-12b353ed3168 kind: Event lastTimestamp: "2026-05-25T12:22:21Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:22:21Z" name: switch-graph-1742e-68c468964c-d9m58.18b2ce665cff50b7 namespace: kserve-ci-e2e-test resourceVersion: "14329" uid: 1f07d39b-4504-4e44-8e85-9e08f27d02cf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1742e} kind: Pod name: switch-graph-1742e-68c468964c-d9m58 namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: cffb6412-1856-40a9-bc55-12b353ed3168 kind: Event lastTimestamp: "2026-05-25T12:22:21Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:41f0ca4b45114a1d81ffad66a667eba2baa4df82ad74c2ffc611dc9cdef0d94a" metadata: creationTimestamp: "2026-05-25T12:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:21Z" name: switch-graph-1742e-68c468964c-d9m58.18b2ce665e46afc1 namespace: kserve-ci-e2e-test resourceVersion: "14331" uid: f3f5ceee-8a98-44a2-adf9-43e408e4453d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1742e} kind: Pod name: switch-graph-1742e-68c468964c-d9m58 namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: cffb6412-1856-40a9-bc55-12b353ed3168 kind: Event lastTimestamp: "2026-05-25T12:22:24Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:41f0ca4b45114a1d81ffad66a667eba2baa4df82ad74c2ffc611dc9cdef0d94a" in 2.61s (2.61s including waiting). Image size: 218005275 bytes.' metadata: creationTimestamp: "2026-05-25T12:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:24Z" name: switch-graph-1742e-68c468964c-d9m58.18b2ce66f9e1db63 namespace: kserve-ci-e2e-test resourceVersion: "14363" uid: eff725d3-b875-4cf0-84a5-c704badd52c5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1742e} kind: Pod name: switch-graph-1742e-68c468964c-d9m58 namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: cffb6412-1856-40a9-bc55-12b353ed3168 kind: Event lastTimestamp: "2026-05-25T12:22:24Z" message: 'Created container: switch-graph-1742e' metadata: creationTimestamp: "2026-05-25T12:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:24Z" name: switch-graph-1742e-68c468964c-d9m58.18b2ce66fd75eff2 namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 98fd6217-5115-4e6a-a483-0b1783b297b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1742e} kind: Pod name: switch-graph-1742e-68c468964c-d9m58 namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: cffb6412-1856-40a9-bc55-12b353ed3168 kind: Event lastTimestamp: "2026-05-25T12:22:24Z" message: Started container switch-graph-1742e metadata: creationTimestamp: "2026-05-25T12:22:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:24Z" name: switch-graph-1742e-68c468964c-d9m58.18b2ce66fde9909c namespace: kserve-ci-e2e-test resourceVersion: "14365" uid: bda15d8b-955d-47c0-bb46-a98c280cb3b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1742e} kind: Pod name: switch-graph-1742e-68c468964c-d9m58 namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: cffb6412-1856-40a9-bc55-12b353ed3168 kind: Event lastTimestamp: "2026-05-25T12:22:30Z" message: Stopping container switch-graph-1742e metadata: creationTimestamp: "2026-05-25T12:22:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:22:30Z" name: switch-graph-1742e-68c468964c-d9m58.18b2ce6878c26951 namespace: kserve-ci-e2e-test resourceVersion: "14443" uid: dbdaf8e7-e72c-448a-9b79-3f7f98ede160 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:22:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1742e} kind: Pod name: switch-graph-1742e-68c468964c-d9m58 namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: cffb6412-1856-40a9-bc55-12b353ed3168 kind: Event lastTimestamp: "2026-05-25T12:23:00Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T12:22:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:00Z" name: switch-graph-1742e-68c468964c-d9m58.18b2ce6995607fe3 namespace: kserve-ci-e2e-test resourceVersion: "14855" uid: 6834a742-17ac-450d-88ba-43c9c30fbbcf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-1742e-68c468964c namespace: kserve-ci-e2e-test resourceVersion: "14301" uid: 951ba53f-90ba-4f2a-aa9c-d55fae23ac25 kind: Event lastTimestamp: "2026-05-25T12:22:20Z" message: 'Created pod: switch-graph-1742e-68c468964c-d9m58' metadata: creationTimestamp: "2026-05-25T12:22:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:20Z" name: switch-graph-1742e-68c468964c.18b2ce661f4d7a24 namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 7dfe6fe6-b88f-4361-b234-c6e73c076f49 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-1742e namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: ee3d2549-a16d-4b73-99d7-09bb86a164c9 kind: Event lastTimestamp: "2026-05-25T12:22:20Z" message: Scaled up replica set switch-graph-1742e-68c468964c from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:22:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:22:20Z" name: switch-graph-1742e.18b2ce661e5e5f48 namespace: kserve-ci-e2e-test resourceVersion: "14302" uid: 23c810e7-6e28-4b8f-8b7b-730688181f2d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:22:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-1742e namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: 063b96e1-a34d-4f00-bd1f-03490e06bc0a kind: Event lastTimestamp: "2026-05-25T12:22:20Z" message: InferenceGraph [switch-graph-1742e] is Ready metadata: creationTimestamp: "2026-05-25T12:22:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:22:20Z" name: switch-graph-1742e.18b2ce66248ba314 namespace: kserve-ci-e2e-test resourceVersion: "14319" uid: 2836f46d-eb32-4448-b963-5c8bea135a4f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-25T12:23:31.117609Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-40bda-7ccd57fd7d-9l88z namespace: kserve-ci-e2e-test resourceVersion: "15316" uid: e8fd5f67-0205-4bc4-bed9-8bd0e53a55a0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-40bda-7ccd57fd7d-9l88z to ip-10-0-143-107.ec2.internal metadata: creationTimestamp: "2026-05-25T12:23:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-25T12:23:31Z" name: switch-graph-40bda-7ccd57fd7d-9l88z.18b2ce76803b92e0 namespace: kserve-ci-e2e-test resourceVersion: "15322" uid: 297eb974-97f5-4159-b933-f1d01f986f2e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-75dcd8bcf6-bn7px source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:31Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-40bda-7ccd57fd7d-9l88z namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: e8fd5f67-0205-4bc4-bed9-8bd0e53a55a0 kind: Event lastTimestamp: "2026-05-25T12:23:31Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-25T12:23:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-25T12:23:31Z" name: switch-graph-40bda-7ccd57fd7d-9l88z.18b2ce769934aad0 namespace: kserve-ci-e2e-test resourceVersion: "15341" uid: c81a3bff-522a-461f-8148-21083bf6d5b4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-40bda} kind: Pod name: switch-graph-40bda-7ccd57fd7d-9l88z namespace: kserve-ci-e2e-test resourceVersion: "15319" uid: e8fd5f67-0205-4bc4-bed9-8bd0e53a55a0 kind: Event lastTimestamp: "2026-05-25T12:23:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:41f0ca4b45114a1d81ffad66a667eba2baa4df82ad74c2ffc611dc9cdef0d94a" already present on machine metadata: creationTimestamp: "2026-05-25T12:23:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:31Z" name: switch-graph-40bda-7ccd57fd7d-9l88z.18b2ce769a8ad273 namespace: kserve-ci-e2e-test resourceVersion: "15344" uid: e1391e27-4ab4-423b-bd55-7c90064c8b49 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-40bda} kind: Pod name: switch-graph-40bda-7ccd57fd7d-9l88z namespace: kserve-ci-e2e-test resourceVersion: "15319" uid: e8fd5f67-0205-4bc4-bed9-8bd0e53a55a0 kind: Event lastTimestamp: "2026-05-25T12:23:31Z" message: 'Created container: switch-graph-40bda' metadata: creationTimestamp: "2026-05-25T12:23:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:31Z" name: switch-graph-40bda-7ccd57fd7d-9l88z.18b2ce769d8fb5ff namespace: kserve-ci-e2e-test resourceVersion: "15345" uid: bcbf86c6-8986-421d-a1e3-f8b4eacf527e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-40bda} kind: Pod name: switch-graph-40bda-7ccd57fd7d-9l88z namespace: kserve-ci-e2e-test resourceVersion: "15319" uid: e8fd5f67-0205-4bc4-bed9-8bd0e53a55a0 kind: Event lastTimestamp: "2026-05-25T12:23:31Z" message: Started container switch-graph-40bda metadata: creationTimestamp: "2026-05-25T12:23:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:23:31Z" name: switch-graph-40bda-7ccd57fd7d-9l88z.18b2ce769e027c63 namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: 0394f88e-b987-4b3c-84dc-2008892bb341 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:31:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-40bda} kind: Pod name: switch-graph-40bda-7ccd57fd7d-9l88z namespace: kserve-ci-e2e-test resourceVersion: "15319" uid: e8fd5f67-0205-4bc4-bed9-8bd0e53a55a0 kind: Event lastTimestamp: "2026-05-25T12:31:45Z" message: Stopping container switch-graph-40bda metadata: creationTimestamp: "2026-05-25T12:31:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:31:45Z" name: switch-graph-40bda-7ccd57fd7d-9l88z.18b2cee9abf2827e namespace: kserve-ci-e2e-test resourceVersion: "18824" uid: a6a6e4c0-a6e4-4805-91b7-36647e277ce5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-25T12:31:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-40bda} kind: Pod name: switch-graph-40bda-7ccd57fd7d-9l88z namespace: kserve-ci-e2e-test resourceVersion: "15319" uid: e8fd5f67-0205-4bc4-bed9-8bd0e53a55a0 kind: Event lastTimestamp: "2026-05-25T12:32:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-25T12:31:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-25T12:32:12Z" name: switch-graph-40bda-7ccd57fd7d-9l88z.18b2ceea26f61838 namespace: kserve-ci-e2e-test resourceVersion: "19153" uid: a7d1f0a8-d5fa-4fff-9669-a45d25add3f5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-107.ec2.internal source: component: kubelet host: ip-10-0-143-107.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-40bda-7ccd57fd7d namespace: kserve-ci-e2e-test resourceVersion: "15312" uid: 61191c54-e6bc-4157-889e-a4a2fb52dcf4 kind: Event lastTimestamp: "2026-05-25T12:23:31Z" message: 'Created pod: switch-graph-40bda-7ccd57fd7d-9l88z' metadata: creationTimestamp: "2026-05-25T12:23:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:23:31Z" name: switch-graph-40bda-7ccd57fd7d.18b2ce767fa7ffb2 namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: a1b96ed4-dd18-400b-95b2-f596662c4909 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-40bda namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: e1a396b9-e0fd-4634-aca1-54631e0c9b5a kind: Event lastTimestamp: "2026-05-25T12:23:31Z" message: Scaled up replica set switch-graph-40bda-7ccd57fd7d from 0 to 1 metadata: creationTimestamp: "2026-05-25T12:23:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:23:31Z" name: switch-graph-40bda.18b2ce767eb8fb11 namespace: kserve-ci-e2e-test resourceVersion: "15313" uid: ee2a5e0f-072d-44af-8a25-234ce99c15f2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-25T12:23:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-40bda namespace: kserve-ci-e2e-test resourceVersion: "15328" uid: f2678ddd-9764-4fb9-a1cb-c61090027e11 kind: Event lastTimestamp: "2026-05-25T12:23:31Z" message: InferenceGraph [switch-graph-40bda] is Ready metadata: creationTimestamp: "2026-05-25T12:23:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-25T12:23:31Z" name: switch-graph-40bda.18b2ce7682c2d5ca namespace: kserve-ci-e2e-test resourceVersion: "15332" uid: a86f613d-583a-4cf7-901d-56a070b5414a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T12:23:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-40bda namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: b3bc5b4b-0f33-47ca-a16c-750009728c0f kind: Event lastTimestamp: "2026-05-25T12:24:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:23:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:24:31Z" name: switch-graph-40bda.18b2ce79fd220940 namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: a82d5db4-0abe-4b21-934f-3e169edc1766 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-25T12:23:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-40bda namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: b3bc5b4b-0f33-47ca-a16c-750009728c0f kind: Event lastTimestamp: "2026-05-25T12:24:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-25T12:23:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-25T12:24:31Z" name: switch-graph-40bda.18b2ce79fd94a763 namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: 85d4437e-5f22-4c4a-bc8e-1fbfc00a684d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "32903"