--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:51:42.330448Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-4aba3-5f6b4f4ff7-vzzq7 namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: 8c8ff629-b2a2-4834-9df1-58b60e87c2e6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-4aba3-5f6b4f4ff7-vzzq7 to ip-10-0-133-161.ec2.internal metadata: creationTimestamp: "2026-04-24T22:51:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:51:42Z" name: ensemble-graph-4aba3-5f6b4f4ff7-vzzq7.18a96cc09304c4b1 namespace: kserve-ci-e2e-test resourceVersion: "20554" uid: ea3d82a6-5b0c-45e7-8797-42bc8f8caff5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:42Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-4aba3-5f6b4f4ff7-vzzq7 namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 8c8ff629-b2a2-4834-9df1-58b60e87c2e6 kind: Event lastTimestamp: "2026-04-24T22:51:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-4aba3-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T22:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:42Z" name: ensemble-graph-4aba3-5f6b4f4ff7-vzzq7.18a96cc09e756473 namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: 8e8df3ab-25d8-49bf-b816-fe7755bfd4e3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:43Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-4aba3-5f6b4f4ff7-vzzq7 namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: 8c8ff629-b2a2-4834-9df1-58b60e87c2e6 kind: Event lastTimestamp: "2026-04-24T22:51:43Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:51:43Z" name: ensemble-graph-4aba3-5f6b4f4ff7-vzzq7.18a96cc0cf8667af namespace: kserve-ci-e2e-test resourceVersion: "20575" uid: c1657372-96d7-47b0-bc7a-42aed4c233b7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4aba3} kind: Pod name: ensemble-graph-4aba3-5f6b4f4ff7-vzzq7 namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 8c8ff629-b2a2-4834-9df1-58b60e87c2e6 kind: Event lastTimestamp: "2026-04-24T22:51:43Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f3ef58ae57fd8b984890d9aeb7935958f0ea100fbddeb0833d76b7e2007120ff" already present on machine metadata: creationTimestamp: "2026-04-24T22:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:43Z" name: ensemble-graph-4aba3-5f6b4f4ff7-vzzq7.18a96cc0d0c4980d namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: 0cfeef1c-0ccb-4477-a0b0-ea2a1735c837 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4aba3} kind: Pod name: ensemble-graph-4aba3-5f6b4f4ff7-vzzq7 namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 8c8ff629-b2a2-4834-9df1-58b60e87c2e6 kind: Event lastTimestamp: "2026-04-24T22:51:43Z" message: 'Created container: ensemble-graph-4aba3' metadata: creationTimestamp: "2026-04-24T22:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:43Z" name: ensemble-graph-4aba3-5f6b4f4ff7-vzzq7.18a96cc0d4323c2c namespace: kserve-ci-e2e-test resourceVersion: "20578" uid: efbce7a0-719c-4215-92ef-9035ee0ec8c8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4aba3} kind: Pod name: ensemble-graph-4aba3-5f6b4f4ff7-vzzq7 namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 8c8ff629-b2a2-4834-9df1-58b60e87c2e6 kind: Event lastTimestamp: "2026-04-24T22:51:43Z" message: Started container ensemble-graph-4aba3 metadata: creationTimestamp: "2026-04-24T22:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:43Z" name: ensemble-graph-4aba3-5f6b4f4ff7-vzzq7.18a96cc0d4a4a616 namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: 0064f03b-b132-4585-85e2-4dfc04381851 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4aba3} kind: Pod name: ensemble-graph-4aba3-5f6b4f4ff7-vzzq7 namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 8c8ff629-b2a2-4834-9df1-58b60e87c2e6 kind: Event lastTimestamp: "2026-04-24T22:59:57Z" message: Stopping container ensemble-graph-4aba3 metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:59:57Z" name: ensemble-graph-4aba3-5f6b4f4ff7-vzzq7.18a96d33c565a268 namespace: kserve-ci-e2e-test resourceVersion: "24079" uid: f94ffb7e-e6f6-474c-974f-e476e48e3681 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:59:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4aba3} kind: Pod name: ensemble-graph-4aba3-5f6b4f4ff7-vzzq7 namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 8c8ff629-b2a2-4834-9df1-58b60e87c2e6 kind: Event lastTimestamp: "2026-04-24T23:00:24Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T22:59:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:24Z" name: ensemble-graph-4aba3-5f6b4f4ff7-vzzq7.18a96d346920c8ce namespace: kserve-ci-e2e-test resourceVersion: "24410" uid: 1b7262c9-7526-48d0-8060-fbac6ea20e02 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-4aba3-5f6b4f4ff7 namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: dab71aff-e1f4-4972-8ee3-80fbb0f9ba70 kind: Event lastTimestamp: "2026-04-24T22:51:42Z" message: 'Created pod: ensemble-graph-4aba3-5f6b4f4ff7-vzzq7' metadata: creationTimestamp: "2026-04-24T22:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:51:42Z" name: ensemble-graph-4aba3-5f6b4f4ff7.18a96cc092963ffe namespace: kserve-ci-e2e-test resourceVersion: "20550" uid: b9a8116d-cf5a-47c0-890f-9e3a31b07338 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-4aba3 namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: 625a47b7-fce7-43c0-950d-c505339646c3 kind: Event lastTimestamp: "2026-04-24T22:51:42Z" message: Scaled up replica set ensemble-graph-4aba3-5f6b4f4ff7 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:51:42Z" name: ensemble-graph-4aba3.18a96cc091f11459 namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: 788a7fd9-3c5f-420a-8689-230614a32b4a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:42Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-4aba3 namespace: kserve-ci-e2e-test resourceVersion: "20560" uid: ab8f045c-0887-4dac-9189-4ae4ff0a483a kind: Event lastTimestamp: "2026-04-24T22:51:42Z" message: InferenceGraph [ensemble-graph-4aba3] is Ready metadata: creationTimestamp: "2026-04-24T22:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:51:42Z" name: ensemble-graph-4aba3.18a96cc095e293be namespace: kserve-ci-e2e-test resourceVersion: "20562" uid: f803c808-b6fb-41c0-9400-efd2bb37171f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:51:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-4aba3 namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: 884e93f4-8cfb-48b2-8388-90179b03826b kind: Event lastTimestamp: "2026-04-24T22:52:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:51:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:52:12Z" name: ensemble-graph-4aba3.18a96cc410612e9c namespace: kserve-ci-e2e-test resourceVersion: "20828" uid: 0150d7e3-a993-4843-a9c0-f9c2644bf432 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:51:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-4aba3 namespace: kserve-ci-e2e-test resourceVersion: "20545" uid: 884e93f4-8cfb-48b2-8388-90179b03826b kind: Event lastTimestamp: "2026-04-24T22:52:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:51:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:52:12Z" name: ensemble-graph-4aba3.18a96cc410e8f859 namespace: kserve-ci-e2e-test resourceVersion: "20829" uid: 85e17416-3e8a-49b5-801b-c87471a02d3b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:50:32.061299Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-5c921-c79bf4c85-pqgsq namespace: kserve-ci-e2e-test resourceVersion: "19584" uid: 6eeabddb-ba1b-4485-bb59-cdfc3f0e0791 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-5c921-c79bf4c85-pqgsq to ip-10-0-133-161.ec2.internal metadata: creationTimestamp: "2026-04-24T22:50:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:50:32Z" name: ensemble-graph-5c921-c79bf4c85-pqgsq.18a96cb036a69d7c namespace: kserve-ci-e2e-test resourceVersion: "19590" uid: 3dda2665-d34f-40c5-ae0f-305c0b2bdf7b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:32Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-5c921-c79bf4c85-pqgsq namespace: kserve-ci-e2e-test resourceVersion: "19593" uid: 6eeabddb-ba1b-4485-bb59-cdfc3f0e0791 kind: Event lastTimestamp: "2026-04-24T22:50:32Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:50:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:50:32Z" name: ensemble-graph-5c921-c79bf4c85-pqgsq.18a96cb04f8610ac namespace: kserve-ci-e2e-test resourceVersion: "19608" uid: da44d0d8-9e8d-476f-a2bd-63783a44aa76 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5c921} kind: Pod name: ensemble-graph-5c921-c79bf4c85-pqgsq namespace: kserve-ci-e2e-test resourceVersion: "19586" uid: 6eeabddb-ba1b-4485-bb59-cdfc3f0e0791 kind: Event lastTimestamp: "2026-04-24T22:50:32Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f3ef58ae57fd8b984890d9aeb7935958f0ea100fbddeb0833d76b7e2007120ff" already present on machine metadata: creationTimestamp: "2026-04-24T22:50:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:32Z" name: ensemble-graph-5c921-c79bf4c85-pqgsq.18a96cb050cbaadd namespace: kserve-ci-e2e-test resourceVersion: "19610" uid: b9ba56d0-23be-47e9-a271-3a0643f2ac89 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5c921} kind: Pod name: ensemble-graph-5c921-c79bf4c85-pqgsq namespace: kserve-ci-e2e-test resourceVersion: "19586" uid: 6eeabddb-ba1b-4485-bb59-cdfc3f0e0791 kind: Event lastTimestamp: "2026-04-24T22:50:32Z" message: 'Created container: ensemble-graph-5c921' metadata: creationTimestamp: "2026-04-24T22:50:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:32Z" name: ensemble-graph-5c921-c79bf4c85-pqgsq.18a96cb0543a0adb namespace: kserve-ci-e2e-test resourceVersion: "19611" uid: 50a48811-f4d7-457c-bc3e-d3e95b3014e4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5c921} kind: Pod name: ensemble-graph-5c921-c79bf4c85-pqgsq namespace: kserve-ci-e2e-test resourceVersion: "19586" uid: 6eeabddb-ba1b-4485-bb59-cdfc3f0e0791 kind: Event lastTimestamp: "2026-04-24T22:50:32Z" message: Started container ensemble-graph-5c921 metadata: creationTimestamp: "2026-04-24T22:50:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:32Z" name: ensemble-graph-5c921-c79bf4c85-pqgsq.18a96cb054a86cb9 namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: a6e19d9f-76fc-44dc-9c91-38e4ba9033fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5c921} kind: Pod name: ensemble-graph-5c921-c79bf4c85-pqgsq namespace: kserve-ci-e2e-test resourceVersion: "19586" uid: 6eeabddb-ba1b-4485-bb59-cdfc3f0e0791 kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: Stopping container ensemble-graph-5c921 metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:42Z" name: ensemble-graph-5c921-c79bf4c85-pqgsq.18a96cb28e97db98 namespace: kserve-ci-e2e-test resourceVersion: "19709" uid: 3067a795-7acf-4c0a-b89d-28abe83692ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:50:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5c921} kind: Pod name: ensemble-graph-5c921-c79bf4c85-pqgsq namespace: kserve-ci-e2e-test resourceVersion: "19586" uid: 6eeabddb-ba1b-4485-bb59-cdfc3f0e0791 kind: Event lastTimestamp: "2026-04-24T22:51:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T22:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:08Z" name: ensemble-graph-5c921-c79bf4c85-pqgsq.18a96cb2e81674ce namespace: kserve-ci-e2e-test resourceVersion: "20059" uid: 52c13d9b-6f2d-45ba-80e1-030c96b62812 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-5c921-c79bf4c85 namespace: kserve-ci-e2e-test resourceVersion: "19581" uid: 9ed6eb33-93bc-4deb-b2cd-731dae8e4baa kind: Event lastTimestamp: "2026-04-24T22:50:32Z" message: 'Created pod: ensemble-graph-5c921-c79bf4c85-pqgsq' metadata: creationTimestamp: "2026-04-24T22:50:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:32Z" name: ensemble-graph-5c921-c79bf4c85.18a96cb0363d8b50 namespace: kserve-ci-e2e-test resourceVersion: "19585" uid: efa1941d-a237-4eb3-b1b9-5bf8adfe7412 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-5c921 namespace: kserve-ci-e2e-test resourceVersion: "19579" uid: caa2aacd-35d1-47ae-a0d7-064ecb1a19ef kind: Event lastTimestamp: "2026-04-24T22:50:32Z" message: Scaled up replica set ensemble-graph-5c921-c79bf4c85 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:50:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:32Z" name: ensemble-graph-5c921.18a96cb035af76b8 namespace: kserve-ci-e2e-test resourceVersion: "19582" uid: 9becb201-0862-44db-8ec7-d37a67403fac reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:32Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-5c921 namespace: kserve-ci-e2e-test resourceVersion: "19597" uid: 46dda3e2-8107-460d-b400-42ceb4020c13 kind: Event lastTimestamp: "2026-04-24T22:50:32Z" message: InferenceGraph [ensemble-graph-5c921] is Ready metadata: creationTimestamp: "2026-04-24T22:50:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:50:32Z" name: ensemble-graph-5c921.18a96cb03974b9dd namespace: kserve-ci-e2e-test resourceVersion: "19599" uid: e9c064ea-bef6-4bca-8568-3b6e82665ff9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:38:47.270628Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv namespace: kserve-ci-e2e-test resourceVersion: "13456" uid: e27eabfc-86ce-49e3-b17b-33d45d7c4352 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-04-24T22:38:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:38:47Z" name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv.18a96c0c1dda6eeb namespace: kserve-ci-e2e-test resourceVersion: "13461" uid: b2a544e5-4ac2-43ea-889c-ad513f83d68d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: e27eabfc-86ce-49e3-b17b-33d45d7c4352 kind: Event lastTimestamp: "2026-04-24T22:38:47Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:38:47Z" name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv.18a96c0c37d2bc1e namespace: kserve-ci-e2e-test resourceVersion: "13486" uid: 8b217152-b8e7-40f0-af7d-ff1fa8a506e8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv namespace: kserve-ci-e2e-test resourceVersion: "13460" uid: e27eabfc-86ce-49e3-b17b-33d45d7c4352 kind: Event lastTimestamp: "2026-04-24T22:38:47Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1443" metadata: creationTimestamp: "2026-04-24T22:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:47Z" name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv.18a96c0c393707c1 namespace: kserve-ci-e2e-test resourceVersion: "13488" uid: fde70bc4-e8af-495f-ab5c-4d9b71acc93a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv namespace: kserve-ci-e2e-test resourceVersion: "13460" uid: e27eabfc-86ce-49e3-b17b-33d45d7c4352 kind: Event lastTimestamp: "2026-04-24T22:39:00Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1443" in 13.026s (13.026s including waiting). Image size: 1334679138 bytes.' metadata: creationTimestamp: "2026-04-24T22:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:00Z" name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv.18a96c0f41a22bd4 namespace: kserve-ci-e2e-test resourceVersion: "13644" uid: d5a484ac-30fb-43be-86b7-f72394eb8eb5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv namespace: kserve-ci-e2e-test resourceVersion: "13460" uid: e27eabfc-86ce-49e3-b17b-33d45d7c4352 kind: Event lastTimestamp: "2026-04-24T22:39:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:00Z" name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv.18a96c0f4687abd0 namespace: kserve-ci-e2e-test resourceVersion: "13646" uid: 48cb71ec-02d2-456b-a774-5eaec854441c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv namespace: kserve-ci-e2e-test resourceVersion: "13460" uid: e27eabfc-86ce-49e3-b17b-33d45d7c4352 kind: Event lastTimestamp: "2026-04-24T22:39:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:00Z" name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv.18a96c0f4706d333 namespace: kserve-ci-e2e-test resourceVersion: "13647" uid: 6058424e-e6de-4ab4-8d57-f9a8d2ed26a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv namespace: kserve-ci-e2e-test resourceVersion: "13460" uid: e27eabfc-86ce-49e3-b17b-33d45d7c4352 kind: Event lastTimestamp: "2026-04-24T22:39:00Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T22:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:00Z" name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv.18a96c0f471307ed namespace: kserve-ci-e2e-test resourceVersion: "13648" uid: 939fba51-628b-44f0-be7e-b482910abd7b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv namespace: kserve-ci-e2e-test resourceVersion: "13460" uid: e27eabfc-86ce-49e3-b17b-33d45d7c4352 kind: Event lastTimestamp: "2026-04-24T22:39:03Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.308s (2.308s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T22:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:03Z" name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv.18a96c0fd0a977ef namespace: kserve-ci-e2e-test resourceVersion: "13698" uid: 66b6220c-5aec-4438-acd4-611983237999 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv namespace: kserve-ci-e2e-test resourceVersion: "13460" uid: e27eabfc-86ce-49e3-b17b-33d45d7c4352 kind: Event lastTimestamp: "2026-04-24T22:39:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:03Z" name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv.18a96c0fd42771a3 namespace: kserve-ci-e2e-test resourceVersion: "13699" uid: 1f36c695-70b6-449f-93c4-e85617e702f1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv namespace: kserve-ci-e2e-test resourceVersion: "13460" uid: e27eabfc-86ce-49e3-b17b-33d45d7c4352 kind: Event lastTimestamp: "2026-04-24T22:39:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:03Z" name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv.18a96c0fd49d57a3 namespace: kserve-ci-e2e-test resourceVersion: "13700" uid: 6bc44c1c-eeba-46a6-8430-b73f915a7a82 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T22:39:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv namespace: kserve-ci-e2e-test resourceVersion: "13460" uid: e27eabfc-86ce-49e3-b17b-33d45d7c4352 kind: Event lastTimestamp: "2026-04-24T22:40:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:19Z" name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv.18a96c0fdcd79ceb namespace: kserve-ci-e2e-test resourceVersion: "14574" uid: d58d00c8-c4f9-4ec5-8abe-5dc19ddf41ca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv namespace: kserve-ci-e2e-test resourceVersion: "13460" uid: e27eabfc-86ce-49e3-b17b-33d45d7c4352 kind: Event lastTimestamp: "2026-04-24T22:40:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:17Z" name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv.18a96c210db32122 namespace: kserve-ci-e2e-test resourceVersion: "14483" uid: 012aec57-ae04-4cad-a724-762c0c3c1f96 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv namespace: kserve-ci-e2e-test resourceVersion: "13460" uid: e27eabfc-86ce-49e3-b17b-33d45d7c4352 kind: Event lastTimestamp: "2026-04-24T22:40:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:17Z" name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv.18a96c210db3a087 namespace: kserve-ci-e2e-test resourceVersion: "14487" uid: bec92718-2636-4b01-9bd4-ba3d17809c2a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv namespace: kserve-ci-e2e-test resourceVersion: "13460" uid: e27eabfc-86ce-49e3-b17b-33d45d7c4352 kind: Event lastTimestamp: "2026-04-24T22:40:19Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:19Z" name: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv.18a96c218ef8a7cf namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: ab81dfa8-0a4f-43be-9df2-a5a2d437006c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1b9dc-predictor-7c988b6956 namespace: kserve-ci-e2e-test resourceVersion: "13436" uid: 26e733da-94a6-439a-a8fd-40360d9c8415 kind: Event lastTimestamp: "2026-04-24T22:38:47Z" message: 'Created pod: error-404-isvc-1b9dc-predictor-7c988b6956-f7lkv' metadata: creationTimestamp: "2026-04-24T22:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:38:47Z" name: error-404-isvc-1b9dc-predictor-7c988b6956.18a96c0c1cc08682 namespace: kserve-ci-e2e-test resourceVersion: "13458" uid: beb70c75-e77c-46c0-a1e9-a33d0c06ed9a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1b9dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "13434" uid: 94011dbb-f3d6-4b60-9757-e1b26d69eea2 kind: Event lastTimestamp: "2026-04-24T22:38:47Z" message: Scaled up replica set error-404-isvc-1b9dc-predictor-7c988b6956 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:38:47Z" name: error-404-isvc-1b9dc-predictor.18a96c0c151ade5c namespace: kserve-ci-e2e-test resourceVersion: "13438" uid: b3329232-e891-4f0d-b45e-cef4d7557180 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:39:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1b9dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "13435" uid: 1cede000-61d8-4886-9be4-616f5d949420 kind: Event lastTimestamp: "2026-04-24T22:39:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:39:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:39:32Z" name: error-404-isvc-1b9dc-predictor.18a96c0f93a1ab47 namespace: kserve-ci-e2e-test resourceVersion: "13982" uid: 3c8e4a30-718b-43c6-bf8e-6c2b8afe13ff reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:39:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1b9dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "13435" uid: 1cede000-61d8-4886-9be4-616f5d949420 kind: Event lastTimestamp: "2026-04-24T22:39:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:39:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:39:32Z" name: error-404-isvc-1b9dc-predictor.18a96c0f9409074d namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: cff12ca6-b9d8-4840-a640-38dece748862 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:39:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1b9dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "13683" uid: 1cede000-61d8-4886-9be4-616f5d949420 kind: Event lastTimestamp: "2026-04-24T22:40:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:02Z" name: error-404-isvc-1b9dc-predictor.18a96c1a10766d9e namespace: kserve-ci-e2e-test resourceVersion: "14237" uid: 4499337f-8b73-44b4-b440-765bf2a0276c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:39:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1b9dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "13683" uid: 1cede000-61d8-4886-9be4-616f5d949420 kind: Event lastTimestamp: "2026-04-24T22:40:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:02Z" name: error-404-isvc-1b9dc-predictor.18a96c1a10e3431a namespace: kserve-ci-e2e-test resourceVersion: "14238" uid: b553c772-9957-4a7c-bcd3-bd47e7991694 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:38:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1b9dc namespace: kserve-ci-e2e-test resourceVersion: "13424" uid: 4927c38f-5bd7-46ec-a0b3-9ac618ab8f84 kind: Event lastTimestamp: "2026-04-24T22:39:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:39:49Z" name: error-404-isvc-1b9dc.18a96c0c15000bf1 namespace: kserve-ci-e2e-test resourceVersion: "14133" uid: ab4d9bcf-966b-4182-9091-8fefec282dd1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1b9dc namespace: kserve-ci-e2e-test resourceVersion: "14131" uid: 4927c38f-5bd7-46ec-a0b3-9ac618ab8f84 kind: Event lastTimestamp: "2026-04-24T22:39:49Z" message: InferenceService [error-404-isvc-1b9dc] is Ready metadata: creationTimestamp: "2026-04-24T22:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:39:49Z" name: error-404-isvc-1b9dc.18a96c1a97e59448 namespace: kserve-ci-e2e-test resourceVersion: "14132" uid: 16d0fa23-5f37-4cd1-b281-50d8b00b3888 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:51:22.297414Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2e6fe-predictor-89f79697d-42755 namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: 4e25e494-c495-44fd-b89c-009820b8b20c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2e6fe-predictor-89f79697d-42755 to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:51:22Z" name: error-404-isvc-2e6fe-predictor-89f79697d-42755.18a96cbbe8f4ed4d namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: f4d3e469-66e6-48a1-8c08-a4f9af09cd2c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2e6fe-predictor-89f79697d-42755 namespace: kserve-ci-e2e-test resourceVersion: "20314" uid: 4e25e494-c495-44fd-b89c-009820b8b20c kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:51:22Z" name: error-404-isvc-2e6fe-predictor-89f79697d-42755.18a96cbc02c78536 namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 60833544-e594-4f0b-b693-49994829d419 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2e6fe-predictor-89f79697d-42755 namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: 4e25e494-c495-44fd-b89c-009820b8b20c kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:22Z" name: error-404-isvc-2e6fe-predictor-89f79697d-42755.18a96cbc042fba62 namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: 0b011ab8-4d08-490e-bfd8-173f7c7deeef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2e6fe-predictor-89f79697d-42755 namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: 4e25e494-c495-44fd-b89c-009820b8b20c kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:22Z" name: error-404-isvc-2e6fe-predictor-89f79697d-42755.18a96cbc0706caa5 namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: d65a3522-ee2c-45c8-aa4b-a7a5fbc5aea9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2e6fe-predictor-89f79697d-42755 namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: 4e25e494-c495-44fd-b89c-009820b8b20c kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:22Z" name: error-404-isvc-2e6fe-predictor-89f79697d-42755.18a96cbc078acf44 namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: 564a82bf-f220-4884-ba5d-05ac374dafc7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2e6fe-predictor-89f79697d-42755 namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: 4e25e494-c495-44fd-b89c-009820b8b20c kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:22Z" name: error-404-isvc-2e6fe-predictor-89f79697d-42755.18a96cbc079845c4 namespace: kserve-ci-e2e-test resourceVersion: "20337" uid: f6cfd749-65aa-4136-b5af-7818597aec30 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2e6fe-predictor-89f79697d-42755 namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: 4e25e494-c495-44fd-b89c-009820b8b20c kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:22Z" name: error-404-isvc-2e6fe-predictor-89f79697d-42755.18a96cbc0ab7f773 namespace: kserve-ci-e2e-test resourceVersion: "20338" uid: b25ea793-1992-49de-b4f6-2413fd63d000 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2e6fe-predictor-89f79697d-42755 namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: 4e25e494-c495-44fd-b89c-009820b8b20c kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:22Z" name: error-404-isvc-2e6fe-predictor-89f79697d-42755.18a96cbc0b2a3bf0 namespace: kserve-ci-e2e-test resourceVersion: "20339" uid: 0f584793-0e8f-4e8e-95b7-3eb72fa53aef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T22:51:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2e6fe-predictor-89f79697d-42755 namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: 4e25e494-c495-44fd-b89c-009820b8b20c kind: Event lastTimestamp: "2026-04-24T23:00:39Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:39Z" name: error-404-isvc-2e6fe-predictor-89f79697d-42755.18a96cbc4acc8304 namespace: kserve-ci-e2e-test resourceVersion: "24665" uid: ea4d85d6-6523-4483-9a53-309d017fb095 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2e6fe-predictor-89f79697d-42755 namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: 4e25e494-c495-44fd-b89c-009820b8b20c kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:37Z" name: error-404-isvc-2e6fe-predictor-89f79697d-42755.18a96d3d19bab86c namespace: kserve-ci-e2e-test resourceVersion: "24590" uid: cad73a05-e0c1-4523-b483-f54bddb57ddb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2e6fe-predictor-89f79697d-42755 namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: 4e25e494-c495-44fd-b89c-009820b8b20c kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:37Z" name: error-404-isvc-2e6fe-predictor-89f79697d-42755.18a96d3d19bbeab9 namespace: kserve-ci-e2e-test resourceVersion: "24594" uid: 243e09c2-8e21-4af6-9153-204c4cb31331 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2e6fe-predictor-89f79697d-42755 namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: 4e25e494-c495-44fd-b89c-009820b8b20c kind: Event lastTimestamp: "2026-04-24T23:00:39Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T23:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:39Z" name: error-404-isvc-2e6fe-predictor-89f79697d-42755.18a96d3dbf2429eb namespace: kserve-ci-e2e-test resourceVersion: "24664" uid: fa07077c-664a-4195-afff-a6e3206b0a04 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2e6fe-predictor-89f79697d namespace: kserve-ci-e2e-test resourceVersion: "20295" uid: f79e9314-180c-40ab-9bff-042aa05667a4 kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: 'Created pod: error-404-isvc-2e6fe-predictor-89f79697d-42755' metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:51:22Z" name: error-404-isvc-2e6fe-predictor-89f79697d.18a96cbbe88ebe32 namespace: kserve-ci-e2e-test resourceVersion: "20304" uid: 783d8a8b-abad-445e-b094-1bc381e153b4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2e6fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 4893a244-b417-4da9-a789-56180d42c87d kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: Scaled up replica set error-404-isvc-2e6fe-predictor-89f79697d from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:51:22Z" name: error-404-isvc-2e6fe-predictor.18a96cbbe70dfed2 namespace: kserve-ci-e2e-test resourceVersion: "20297" uid: ea773482-3b32-4431-8c09-a539f0909634 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:51:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2e6fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 0db2e150-263c-443d-af3b-61daf0d0c1cd kind: Event lastTimestamp: "2026-04-24T22:52:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:51:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:52:07Z" name: error-404-isvc-2e6fe-predictor.18a96cbf658e38f2 namespace: kserve-ci-e2e-test resourceVersion: "20774" uid: 36d7d71a-6cf6-4d4e-9005-d1baa54debc4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:51:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2e6fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: 0db2e150-263c-443d-af3b-61daf0d0c1cd kind: Event lastTimestamp: "2026-04-24T22:52:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:51:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:52:07Z" name: error-404-isvc-2e6fe-predictor.18a96cbf65ea2c7c namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: d8edaf96-0bc3-4517-9c06-4efe98c874b8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:51:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2e6fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20493" uid: 0db2e150-263c-443d-af3b-61daf0d0c1cd kind: Event lastTimestamp: "2026-04-24T22:52:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:52:37Z" name: error-404-isvc-2e6fe-predictor.18a96cc2e496ce52 namespace: kserve-ci-e2e-test resourceVersion: "21056" uid: 6006753c-cc5e-40fa-bc42-cf88656a905a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:51:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2e6fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20493" uid: 0db2e150-263c-443d-af3b-61daf0d0c1cd kind: Event lastTimestamp: "2026-04-24T22:52:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:52:37Z" name: error-404-isvc-2e6fe-predictor.18a96cc2e4f8e26b namespace: kserve-ci-e2e-test resourceVersion: "21057" uid: fe361a49-12ae-44d0-b721-24310f2a57ec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2e6fe namespace: kserve-ci-e2e-test resourceVersion: "20284" uid: b5a0aaf2-6890-4ba0-a2dd-95c18d53f2f1 kind: Event lastTimestamp: "2026-04-24T22:52:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:52:10Z" name: error-404-isvc-2e6fe.18a96cbbe6e9abbd namespace: kserve-ci-e2e-test resourceVersion: "20803" uid: 20541a3e-c79b-4f32-ae85-aa82c00bb51a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2e6fe namespace: kserve-ci-e2e-test resourceVersion: "20284" uid: b5a0aaf2-6890-4ba0-a2dd-95c18d53f2f1 kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: 'Failed to update status for InferenceService "error-404-isvc-2e6fe": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2e6fe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:51:22Z" name: error-404-isvc-2e6fe.18a96cbbe73be31a namespace: kserve-ci-e2e-test resourceVersion: "20299" uid: 2e8f81b7-bc97-4e6d-a611-7b14db9ab3b8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2e6fe namespace: kserve-ci-e2e-test resourceVersion: "20284" uid: b5a0aaf2-6890-4ba0-a2dd-95c18d53f2f1 kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2e6fe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:51:22Z" name: error-404-isvc-2e6fe.18a96cbbe73c4f6c namespace: kserve-ci-e2e-test resourceVersion: "20300" uid: 802d03d7-78cc-4765-9fce-ca863b436218 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:52:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2e6fe namespace: kserve-ci-e2e-test resourceVersion: "20800" uid: b5a0aaf2-6890-4ba0-a2dd-95c18d53f2f1 kind: Event lastTimestamp: "2026-04-24T22:52:10Z" message: InferenceService [error-404-isvc-2e6fe] is Ready metadata: creationTimestamp: "2026-04-24T22:52:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:52:10Z" name: error-404-isvc-2e6fe.18a96cc705bff0fa namespace: kserve-ci-e2e-test resourceVersion: "20802" uid: b57e1cb5-a698-4bfc-89d0-049df6cd3077 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:50:12.051815Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z namespace: kserve-ci-e2e-test resourceVersion: "19344" uid: 675b0d26-e528-409f-951e-a218d9d2b936 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-04-24T22:50:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:50:12Z" name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z.18a96cab8dfe2cd7 namespace: kserve-ci-e2e-test resourceVersion: "19348" uid: def233f3-e813-4e7a-b8cd-000ea1a9d62e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z namespace: kserve-ci-e2e-test resourceVersion: "19345" uid: 675b0d26-e528-409f-951e-a218d9d2b936 kind: Event lastTimestamp: "2026-04-24T22:50:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-32c7c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T22:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:12Z" name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z.18a96cab979aa7e2 namespace: kserve-ci-e2e-test resourceVersion: "19356" uid: 629752f3-5361-40c9-a3da-2e83a76d3031 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z namespace: kserve-ci-e2e-test resourceVersion: "19354" uid: 675b0d26-e528-409f-951e-a218d9d2b936 kind: Event lastTimestamp: "2026-04-24T22:50:13Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:50:13Z" name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z.18a96cabcb7a6ec4 namespace: kserve-ci-e2e-test resourceVersion: "19378" uid: fea883f0-d659-4e5b-89df-dec108063105 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z namespace: kserve-ci-e2e-test resourceVersion: "19345" uid: 675b0d26-e528-409f-951e-a218d9d2b936 kind: Event lastTimestamp: "2026-04-24T22:50:13Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:13Z" name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z.18a96cabccc2bf8c namespace: kserve-ci-e2e-test resourceVersion: "19380" uid: 5784cd2a-8255-4744-8c69-367899ead6f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z namespace: kserve-ci-e2e-test resourceVersion: "19345" uid: 675b0d26-e528-409f-951e-a218d9d2b936 kind: Event lastTimestamp: "2026-04-24T22:50:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:13Z" name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z.18a96cabcfd22f98 namespace: kserve-ci-e2e-test resourceVersion: "19384" uid: 7902dea1-a3a5-41cf-b517-caa146986d72 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z namespace: kserve-ci-e2e-test resourceVersion: "19345" uid: 675b0d26-e528-409f-951e-a218d9d2b936 kind: Event lastTimestamp: "2026-04-24T22:50:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:13Z" name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z.18a96cabd048b3ee namespace: kserve-ci-e2e-test resourceVersion: "19385" uid: 25ba7bf4-b160-41a4-810e-cfe20636cc7a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z namespace: kserve-ci-e2e-test resourceVersion: "19345" uid: 675b0d26-e528-409f-951e-a218d9d2b936 kind: Event lastTimestamp: "2026-04-24T22:50:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:13Z" name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z.18a96cabd0560f11 namespace: kserve-ci-e2e-test resourceVersion: "19386" uid: 9550766e-07f8-4d39-9bc9-596c7dd2e3c4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z namespace: kserve-ci-e2e-test resourceVersion: "19345" uid: 675b0d26-e528-409f-951e-a218d9d2b936 kind: Event lastTimestamp: "2026-04-24T22:50:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:13Z" name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z.18a96cabd4417ac7 namespace: kserve-ci-e2e-test resourceVersion: "19388" uid: 75d6a3f0-f739-40a4-b63a-1476893f7ffe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z namespace: kserve-ci-e2e-test resourceVersion: "19345" uid: 675b0d26-e528-409f-951e-a218d9d2b936 kind: Event lastTimestamp: "2026-04-24T22:50:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:13Z" name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z.18a96cabd4b6ab55 namespace: kserve-ci-e2e-test resourceVersion: "19389" uid: 2e6b55d7-5973-4ae2-b7ca-ca9c1af76c89 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z namespace: kserve-ci-e2e-test resourceVersion: "19345" uid: 675b0d26-e528-409f-951e-a218d9d2b936 kind: Event lastTimestamp: "2026-04-24T22:50:50Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:50Z" name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z.18a96cac2a6ca3ba namespace: kserve-ci-e2e-test resourceVersion: "19905" uid: 6cf4adf8-3fbc-4b56-82df-c726e7fea8a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z namespace: kserve-ci-e2e-test resourceVersion: "19345" uid: 675b0d26-e528-409f-951e-a218d9d2b936 kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:22Z" name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z.18a96cbbe3331c01 namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: d7b888fd-99d0-4c5b-9487-3cc7d4a1ed63 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z namespace: kserve-ci-e2e-test resourceVersion: "19345" uid: 675b0d26-e528-409f-951e-a218d9d2b936 kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:22Z" name: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z.18a96cbbe3344717 namespace: kserve-ci-e2e-test resourceVersion: "20283" uid: 3b88ca18-c653-402d-88f0-c7ee5697ceda reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-32c7c-predictor-69fdd96fd6 namespace: kserve-ci-e2e-test resourceVersion: "19337" uid: ab2b12dd-9f9c-4d89-a514-e71c1d9118d6 kind: Event lastTimestamp: "2026-04-24T22:50:12Z" message: 'Created pod: error-404-isvc-32c7c-predictor-69fdd96fd6-v6c7z' metadata: creationTimestamp: "2026-04-24T22:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:12Z" name: error-404-isvc-32c7c-predictor-69fdd96fd6.18a96cab8db01986 namespace: kserve-ci-e2e-test resourceVersion: "19346" uid: 4e527321-0b1b-49cd-ab20-d4914525e648 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-32c7c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19334" uid: 1127c1fe-9c57-4722-b9a9-e1cf6511216d kind: Event lastTimestamp: "2026-04-24T22:50:12Z" message: Scaled up replica set error-404-isvc-32c7c-predictor-69fdd96fd6 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:12Z" name: error-404-isvc-32c7c-predictor.18a96cab8c077049 namespace: kserve-ci-e2e-test resourceVersion: "19338" uid: 2e26c015-4de8-4259-9413-8aeabbbda408 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:50:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32c7c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19336" uid: 058bf53f-dc47-4cd4-bfba-02697996c62e kind: Event lastTimestamp: "2026-04-24T22:50:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:57Z" name: error-404-isvc-32c7c-predictor.18a96caf0ac09835 namespace: kserve-ci-e2e-test resourceVersion: "19950" uid: 8c1c668a-adb0-4e8f-80d2-0d615261e2a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:50:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32c7c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19336" uid: 058bf53f-dc47-4cd4-bfba-02697996c62e kind: Event lastTimestamp: "2026-04-24T22:50:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:57Z" name: error-404-isvc-32c7c-predictor.18a96caf0b48e2ed namespace: kserve-ci-e2e-test resourceVersion: "19951" uid: c552b6bb-c45a-424f-a0d0-9dba47895547 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32c7c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19530" uid: 058bf53f-dc47-4cd4-bfba-02697996c62e kind: Event lastTimestamp: "2026-04-24T22:51:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:51:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:51:12Z" name: error-404-isvc-32c7c-predictor.18a96cb98768119c namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: d5e056f2-fa67-478d-855c-4966067b26ce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32c7c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19530" uid: 058bf53f-dc47-4cd4-bfba-02697996c62e kind: Event lastTimestamp: "2026-04-24T22:51:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:51:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:51:12Z" name: error-404-isvc-32c7c-predictor.18a96cb987ad03b4 namespace: kserve-ci-e2e-test resourceVersion: "20119" uid: fc147208-d1d0-42ef-91ab-531dc9dc2b57 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:50:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-32c7c namespace: kserve-ci-e2e-test resourceVersion: "19305" uid: 2f0b6d09-2f5a-4b0f-a098-1596034d4144 kind: Event lastTimestamp: "2026-04-24T22:51:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:51:00Z" name: error-404-isvc-32c7c.18a96cab8c0da7a8 namespace: kserve-ci-e2e-test resourceVersion: "19994" uid: 8e1d3f9c-9560-459c-adb0-530c8fddc326 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-32c7c namespace: kserve-ci-e2e-test resourceVersion: "19305" uid: 2f0b6d09-2f5a-4b0f-a098-1596034d4144 kind: Event lastTimestamp: "2026-04-24T22:50:12Z" message: 'Failed to update status for InferenceService "error-404-isvc-32c7c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-32c7c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:50:12Z" name: error-404-isvc-32c7c.18a96cab8c94b7b8 namespace: kserve-ci-e2e-test resourceVersion: "19341" uid: 4c484b48-744c-4110-a307-4d1b004d1980 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-32c7c namespace: kserve-ci-e2e-test resourceVersion: "19305" uid: 2f0b6d09-2f5a-4b0f-a098-1596034d4144 kind: Event lastTimestamp: "2026-04-24T22:50:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-32c7c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:50:12Z" name: error-404-isvc-32c7c.18a96cab8c950cbf namespace: kserve-ci-e2e-test resourceVersion: "19342" uid: 6cc28c0a-b4e8-4f85-bfe1-dd353f936c60 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-32c7c namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 2f0b6d09-2f5a-4b0f-a098-1596034d4144 kind: Event lastTimestamp: "2026-04-24T22:51:00Z" message: InferenceService [error-404-isvc-32c7c] is Ready metadata: creationTimestamp: "2026-04-24T22:51:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:51:00Z" name: error-404-isvc-32c7c.18a96cb6e567b621 namespace: kserve-ci-e2e-test resourceVersion: "19993" uid: df4ad33b-a619-45f5-90e0-080491ff722d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:59:57.319815Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n namespace: kserve-ci-e2e-test resourceVersion: "24154" uid: de53ce31-1453-4f66-9a58-bf6080a80a42 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:59:57Z" name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n.18a96d33d2af47d2 namespace: kserve-ci-e2e-test resourceVersion: "24158" uid: cef5ca06-5543-460b-848d-13e5bf7066c0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n namespace: kserve-ci-e2e-test resourceVersion: "24164" uid: de53ce31-1453-4f66-9a58-bf6080a80a42 kind: Event lastTimestamp: "2026-04-24T22:59:57Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:59:57Z" name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n.18a96d33ec3ca535 namespace: kserve-ci-e2e-test resourceVersion: "24174" uid: 95ef79cb-d476-4190-9ce4-2d407a4c09e1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n namespace: kserve-ci-e2e-test resourceVersion: "24155" uid: de53ce31-1453-4f66-9a58-bf6080a80a42 kind: Event lastTimestamp: "2026-04-24T22:59:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:59:57Z" name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n.18a96d33ed778808 namespace: kserve-ci-e2e-test resourceVersion: "24176" uid: e94f42d5-4383-4dcd-9771-0727a9a50bbc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n namespace: kserve-ci-e2e-test resourceVersion: "24155" uid: de53ce31-1453-4f66-9a58-bf6080a80a42 kind: Event lastTimestamp: "2026-04-24T22:59:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:59:57Z" name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n.18a96d33f0cb7301 namespace: kserve-ci-e2e-test resourceVersion: "24177" uid: f91dd17f-baca-4606-9e0a-1ed78e4fd906 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n namespace: kserve-ci-e2e-test resourceVersion: "24155" uid: de53ce31-1453-4f66-9a58-bf6080a80a42 kind: Event lastTimestamp: "2026-04-24T22:59:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:59:57Z" name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n.18a96d33f1425930 namespace: kserve-ci-e2e-test resourceVersion: "24179" uid: 3b797031-358e-4e4d-9e3f-a5747e212d73 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n namespace: kserve-ci-e2e-test resourceVersion: "24155" uid: de53ce31-1453-4f66-9a58-bf6080a80a42 kind: Event lastTimestamp: "2026-04-24T22:59:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:59:57Z" name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n.18a96d33f14e2ee1 namespace: kserve-ci-e2e-test resourceVersion: "24180" uid: 78d0be6f-0783-4296-bd1c-9077e5690a84 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n namespace: kserve-ci-e2e-test resourceVersion: "24155" uid: de53ce31-1453-4f66-9a58-bf6080a80a42 kind: Event lastTimestamp: "2026-04-24T22:59:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:59:57Z" name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n.18a96d33f44c0e17 namespace: kserve-ci-e2e-test resourceVersion: "24181" uid: d09c0b2d-1844-43b8-9214-a4e6744cdefb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n namespace: kserve-ci-e2e-test resourceVersion: "24155" uid: de53ce31-1453-4f66-9a58-bf6080a80a42 kind: Event lastTimestamp: "2026-04-24T22:59:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:59:57Z" name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n.18a96d33f4d4c4ea namespace: kserve-ci-e2e-test resourceVersion: "24182" uid: df3dd699-5d1e-4ee7-9814-e1358d553d16 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:59:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n namespace: kserve-ci-e2e-test resourceVersion: "24155" uid: de53ce31-1453-4f66-9a58-bf6080a80a42 kind: Event lastTimestamp: "2026-04-24T23:00:35Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:59:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:35Z" name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n.18a96d34629ab080 namespace: kserve-ci-e2e-test resourceVersion: "24500" uid: 8bbfbd37-d285-4ffd-9012-873e8de22b7e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n namespace: kserve-ci-e2e-test resourceVersion: "24155" uid: de53ce31-1453-4f66-9a58-bf6080a80a42 kind: Event lastTimestamp: "2026-04-24T23:01:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:07Z" name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n.18a96d442a83c5dc namespace: kserve-ci-e2e-test resourceVersion: "25000" uid: cf013986-609a-4ab7-9043-38c2e6e60dbb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n namespace: kserve-ci-e2e-test resourceVersion: "24155" uid: de53ce31-1453-4f66-9a58-bf6080a80a42 kind: Event lastTimestamp: "2026-04-24T23:01:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:07Z" name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n.18a96d442a87a35e namespace: kserve-ci-e2e-test resourceVersion: "25005" uid: 2dd26991-28bc-494a-91ab-27ccb87d3903 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n namespace: kserve-ci-e2e-test resourceVersion: "24155" uid: de53ce31-1453-4f66-9a58-bf6080a80a42 kind: Event lastTimestamp: "2026-04-24T23:01:10Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T23:01:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:10Z" name: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n.18a96d44ead0584f namespace: kserve-ci-e2e-test resourceVersion: "25123" uid: 3a8079d9-53d1-4d9d-ae54-3965cca1226d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-44b1e-predictor-648b8cfbf8 namespace: kserve-ci-e2e-test resourceVersion: "24144" uid: 674e1a04-6dcb-4ecc-a883-e315a7338dcc kind: Event lastTimestamp: "2026-04-24T22:59:57Z" message: 'Created pod: error-404-isvc-44b1e-predictor-648b8cfbf8-mdq6n' metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:59:57Z" name: error-404-isvc-44b1e-predictor-648b8cfbf8.18a96d33d26df2af namespace: kserve-ci-e2e-test resourceVersion: "24156" uid: 63c2a5da-eed6-4575-84cf-518b370afc35 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-44b1e-predictor namespace: kserve-ci-e2e-test resourceVersion: "24140" uid: 86d18ebe-ed60-47a8-8d7f-1cad189b1e95 kind: Event lastTimestamp: "2026-04-24T22:59:57Z" message: Scaled up replica set error-404-isvc-44b1e-predictor-648b8cfbf8 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:59:57Z" name: error-404-isvc-44b1e-predictor.18a96d33d0e981e7 namespace: kserve-ci-e2e-test resourceVersion: "24148" uid: 250458ba-eba9-4cfb-a2dc-f28efe210e45 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T23:00:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-44b1e-predictor namespace: kserve-ci-e2e-test resourceVersion: "24143" uid: 94462923-759f-4e75-ae4b-20a831bfb2de kind: Event lastTimestamp: "2026-04-24T23:00:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T23:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:00:27Z" name: error-404-isvc-44b1e-predictor.18a96d374f66f9ac namespace: kserve-ci-e2e-test resourceVersion: "24430" uid: fc34545d-f54c-4409-b9b7-cdfead363353 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T23:00:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-44b1e-predictor namespace: kserve-ci-e2e-test resourceVersion: "24143" uid: 94462923-759f-4e75-ae4b-20a831bfb2de kind: Event lastTimestamp: "2026-04-24T23:00:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T23:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:00:27Z" name: error-404-isvc-44b1e-predictor.18a96d374fca2288 namespace: kserve-ci-e2e-test resourceVersion: "24431" uid: fabb97c6-95e7-4cee-8eae-60165ec8000a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T23:00:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-44b1e-predictor namespace: kserve-ci-e2e-test resourceVersion: "24322" uid: 94462923-759f-4e75-ae4b-20a831bfb2de kind: Event lastTimestamp: "2026-04-24T23:00:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T23:00:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:00:57Z" name: error-404-isvc-44b1e-predictor.18a96d3e4d30b1c0 namespace: kserve-ci-e2e-test resourceVersion: "24853" uid: d0050607-3bb4-4201-aaa4-87659e8ed147 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T23:00:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-44b1e-predictor namespace: kserve-ci-e2e-test resourceVersion: "24322" uid: 94462923-759f-4e75-ae4b-20a831bfb2de kind: Event lastTimestamp: "2026-04-24T23:00:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T23:00:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:00:57Z" name: error-404-isvc-44b1e-predictor.18a96d3e4d9dab4b namespace: kserve-ci-e2e-test resourceVersion: "24856" uid: 521f3966-50f5-4b19-a3ee-8c5523a53785 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-44b1e namespace: kserve-ci-e2e-test resourceVersion: "24121" uid: e51324ad-3ea5-4520-be29-b3cf472dfc82 kind: Event lastTimestamp: "2026-04-24T23:00:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:00:45Z" name: error-404-isvc-44b1e.18a96d33d0c7a87e namespace: kserve-ci-e2e-test resourceVersion: "24746" uid: 0243ecbc-3471-4963-a3cb-70110fb738fc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-44b1e namespace: kserve-ci-e2e-test resourceVersion: "24121" uid: e51324ad-3ea5-4520-be29-b3cf472dfc82 kind: Event lastTimestamp: "2026-04-24T22:59:57Z" message: 'Failed to update status for InferenceService "error-404-isvc-44b1e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-44b1e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:59:57Z" name: error-404-isvc-44b1e.18a96d33d16723a0 namespace: kserve-ci-e2e-test resourceVersion: "24150" uid: 88289d37-bdde-4077-a84b-d2e9ae4e4ad9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-44b1e namespace: kserve-ci-e2e-test resourceVersion: "24121" uid: e51324ad-3ea5-4520-be29-b3cf472dfc82 kind: Event lastTimestamp: "2026-04-24T22:59:57Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-44b1e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:59:57Z" name: error-404-isvc-44b1e.18a96d33d16795f0 namespace: kserve-ci-e2e-test resourceVersion: "24152" uid: 9680216d-d385-42c6-830d-cbea2042ee44 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-44b1e namespace: kserve-ci-e2e-test resourceVersion: "24743" uid: e51324ad-3ea5-4520-be29-b3cf472dfc82 kind: Event lastTimestamp: "2026-04-24T23:00:45Z" message: InferenceService [error-404-isvc-44b1e] is Ready metadata: creationTimestamp: "2026-04-24T23:00:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:00:45Z" name: error-404-isvc-44b1e.18a96d3f1d83ad01 namespace: kserve-ci-e2e-test resourceVersion: "24744" uid: 77827269-ad26-467d-a243-93f3f7251541 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:50:42.412346Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw namespace: kserve-ci-e2e-test resourceVersion: "19786" uid: 5aed5c0b-de0f-4bf1-b1e7-bf379bb3ee21 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4aba3-predictor-6f567c46fb-kdltw to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:50:42Z" name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw.18a96cb29f9f1360 namespace: kserve-ci-e2e-test resourceVersion: "19790" uid: 10c9464c-368c-44f1-9e92-651ef8fc94e0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: 5aed5c0b-de0f-4bf1-b1e7-bf379bb3ee21 kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-4aba3-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:42Z" name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw.18a96cb2afdbd50e namespace: kserve-ci-e2e-test resourceVersion: "19806" uid: 091e9ba1-1d07-4e7c-992f-37133f1aecba reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:43Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw namespace: kserve-ci-e2e-test resourceVersion: "19797" uid: 5aed5c0b-de0f-4bf1-b1e7-bf379bb3ee21 kind: Event lastTimestamp: "2026-04-24T22:50:43Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:50:43Z" name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw.18a96cb2dcebff2e namespace: kserve-ci-e2e-test resourceVersion: "19828" uid: c6b6b7b3-bb27-4506-9a6d-1ecbec3c13ef reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: 5aed5c0b-de0f-4bf1-b1e7-bf379bb3ee21 kind: Event lastTimestamp: "2026-04-24T22:50:43Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:43Z" name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw.18a96cb2de26eb34 namespace: kserve-ci-e2e-test resourceVersion: "19830" uid: 35a74d68-cc55-47c6-8eed-ece2d0bc6789 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: 5aed5c0b-de0f-4bf1-b1e7-bf379bb3ee21 kind: Event lastTimestamp: "2026-04-24T22:50:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:43Z" name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw.18a96cb2e1346b9d namespace: kserve-ci-e2e-test resourceVersion: "19831" uid: ccd843c6-11a1-4031-8948-af5a1b56cb43 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: 5aed5c0b-de0f-4bf1-b1e7-bf379bb3ee21 kind: Event lastTimestamp: "2026-04-24T22:50:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:43Z" name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw.18a96cb2e1ab45c3 namespace: kserve-ci-e2e-test resourceVersion: "19832" uid: 691aab2b-489d-4510-96bd-1615a0ce4c47 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: 5aed5c0b-de0f-4bf1-b1e7-bf379bb3ee21 kind: Event lastTimestamp: "2026-04-24T22:50:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:43Z" name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw.18a96cb2e1b72fa7 namespace: kserve-ci-e2e-test resourceVersion: "19833" uid: eadf1567-f63f-4c1b-bee8-9223a1b9ae81 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: 5aed5c0b-de0f-4bf1-b1e7-bf379bb3ee21 kind: Event lastTimestamp: "2026-04-24T22:50:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:43Z" name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw.18a96cb2e50d6bcb namespace: kserve-ci-e2e-test resourceVersion: "19834" uid: 55d8a24f-9ef0-4b72-83f9-3d49d859b591 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: 5aed5c0b-de0f-4bf1-b1e7-bf379bb3ee21 kind: Event lastTimestamp: "2026-04-24T22:50:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:50:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:43Z" name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw.18a96cb2e588527e namespace: kserve-ci-e2e-test resourceVersion: "19835" uid: c6c81cda-3edd-405d-a54e-ed67073f5cfd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:50:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: 5aed5c0b-de0f-4bf1-b1e7-bf379bb3ee21 kind: Event lastTimestamp: "2026-04-24T22:51:20Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:20Z" name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw.18a96cb32d44596f namespace: kserve-ci-e2e-test resourceVersion: "20200" uid: 9fb47adc-8053-4d57-bed9-f098d8687e1a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: 5aed5c0b-de0f-4bf1-b1e7-bf379bb3ee21 kind: Event lastTimestamp: "2026-04-24T22:59:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:59:57Z" name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw.18a96d33ce685369 namespace: kserve-ci-e2e-test resourceVersion: "24126" uid: a36e6d5d-b8c6-45cd-90da-030444f5b4ec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw namespace: kserve-ci-e2e-test resourceVersion: "19788" uid: 5aed5c0b-de0f-4bf1-b1e7-bf379bb3ee21 kind: Event lastTimestamp: "2026-04-24T22:59:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:59:57Z" name: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw.18a96d33ce68ca61 namespace: kserve-ci-e2e-test resourceVersion: "24128" uid: af748dde-8fd8-46fc-a119-a1f6620935b3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4aba3-predictor-6f567c46fb namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 4ac98e57-d29e-4ea8-8cd2-5970ab0ad997 kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: 'Created pod: error-404-isvc-4aba3-predictor-6f567c46fb-kdltw' metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:42Z" name: error-404-isvc-4aba3-predictor-6f567c46fb.18a96cb29f3c231b namespace: kserve-ci-e2e-test resourceVersion: "19787" uid: befee31d-4590-446f-b8b2-ab0c196b82fa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4aba3-predictor namespace: kserve-ci-e2e-test resourceVersion: "19775" uid: 5dd0d7c5-f32e-47c8-ba52-c605c20bd14a kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: Scaled up replica set error-404-isvc-4aba3-predictor-6f567c46fb from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:42Z" name: error-404-isvc-4aba3-predictor.18a96cb29d6c68a1 namespace: kserve-ci-e2e-test resourceVersion: "19780" uid: 85632797-8a81-4ea2-80f3-964127293784 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:50:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4aba3-predictor namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: 99bc1b8f-4363-47e0-9513-7f932d7c7366 kind: Event lastTimestamp: "2026-04-24T22:51:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:51:12Z" name: error-404-isvc-4aba3-predictor.18a96cb61be07e7b namespace: kserve-ci-e2e-test resourceVersion: "20126" uid: 3b8fb6ed-763e-421b-ba8d-45dd07404c32 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:50:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4aba3-predictor namespace: kserve-ci-e2e-test resourceVersion: "19776" uid: 99bc1b8f-4363-47e0-9513-7f932d7c7366 kind: Event lastTimestamp: "2026-04-24T22:51:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:51:12Z" name: error-404-isvc-4aba3-predictor.18a96cb61c367b11 namespace: kserve-ci-e2e-test resourceVersion: "20127" uid: 678c86b3-4b6b-40d6-830f-bab6f74fc062 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:51:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4aba3-predictor namespace: kserve-ci-e2e-test resourceVersion: "19958" uid: 99bc1b8f-4363-47e0-9513-7f932d7c7366 kind: Event lastTimestamp: "2026-04-24T22:51:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:51:57Z" name: error-404-isvc-4aba3-predictor.18a96cbd19adacfa namespace: kserve-ci-e2e-test resourceVersion: "20702" uid: 9b83ddd6-e53a-491e-9f7f-ddebd8cbefa5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:51:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4aba3-predictor namespace: kserve-ci-e2e-test resourceVersion: "19958" uid: 99bc1b8f-4363-47e0-9513-7f932d7c7366 kind: Event lastTimestamp: "2026-04-24T22:51:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:51:57Z" name: error-404-isvc-4aba3-predictor.18a96cbd19f4f07c namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: f01b09b9-3ec9-446f-a70b-6214b47d61bc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4aba3 namespace: kserve-ci-e2e-test resourceVersion: "19746" uid: 92c56d95-1eb6-4e88-8e88-a5a0926635a3 kind: Event lastTimestamp: "2026-04-24T22:51:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:51:30Z" name: error-404-isvc-4aba3.18a96cb29d469d2e namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: dccdd1dd-2fea-4adc-a978-5173803a5d50 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4aba3 namespace: kserve-ci-e2e-test resourceVersion: "19746" uid: 92c56d95-1eb6-4e88-8e88-a5a0926635a3 kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: 'Failed to update status for InferenceService "error-404-isvc-4aba3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4aba3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:50:42Z" name: error-404-isvc-4aba3.18a96cb29db2cc76 namespace: kserve-ci-e2e-test resourceVersion: "19783" uid: 5f5b42cf-028b-4d58-8758-09973aa35afb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4aba3 namespace: kserve-ci-e2e-test resourceVersion: "19746" uid: 92c56d95-1eb6-4e88-8e88-a5a0926635a3 kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4aba3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:50:42Z" name: error-404-isvc-4aba3.18a96cb29db33b5a namespace: kserve-ci-e2e-test resourceVersion: "19784" uid: df19b318-aaeb-4334-a861-88269bb19989 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4aba3 namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: 92c56d95-1eb6-4e88-8e88-a5a0926635a3 kind: Event lastTimestamp: "2026-04-24T22:51:30Z" message: InferenceService [error-404-isvc-4aba3] is Ready metadata: creationTimestamp: "2026-04-24T22:51:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:51:30Z" name: error-404-isvc-4aba3.18a96cbde9974d75 namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: b0630a15-d5ed-445b-8291-490f66a76d36 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:49:32.053526Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn namespace: kserve-ci-e2e-test resourceVersion: "18899" uid: c3f98fde-f0aa-415a-a8a4-02c829157a8f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-04-24T22:49:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:49:32Z" name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn.18a96ca23de8b310 namespace: kserve-ci-e2e-test resourceVersion: "18903" uid: 36911b13-3a1b-42b0-8af4-c038fa039fdc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn namespace: kserve-ci-e2e-test resourceVersion: "18910" uid: c3f98fde-f0aa-415a-a8a4-02c829157a8f kind: Event lastTimestamp: "2026-04-24T22:49:32Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:49:32Z" name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn.18a96ca25785a189 namespace: kserve-ci-e2e-test resourceVersion: "18921" uid: b583d368-e144-4c78-b2ac-beb9c63ef717 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: c3f98fde-f0aa-415a-a8a4-02c829157a8f kind: Event lastTimestamp: "2026-04-24T22:49:32Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:32Z" name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn.18a96ca258e1fbb5 namespace: kserve-ci-e2e-test resourceVersion: "18923" uid: b853eeff-106e-4816-b75d-b563a06f1152 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: c3f98fde-f0aa-415a-a8a4-02c829157a8f kind: Event lastTimestamp: "2026-04-24T22:49:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:32Z" name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn.18a96ca25c1ff0a8 namespace: kserve-ci-e2e-test resourceVersion: "18924" uid: d70f4ef8-1ef5-4406-be6e-9c05d37c2174 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: c3f98fde-f0aa-415a-a8a4-02c829157a8f kind: Event lastTimestamp: "2026-04-24T22:49:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:32Z" name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn.18a96ca25c9e47e1 namespace: kserve-ci-e2e-test resourceVersion: "18925" uid: 61dfdd49-d624-4c71-9dda-456fe0ac6f3e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: c3f98fde-f0aa-415a-a8a4-02c829157a8f kind: Event lastTimestamp: "2026-04-24T22:49:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:32Z" name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn.18a96ca25ca91b3c namespace: kserve-ci-e2e-test resourceVersion: "18926" uid: 099735d0-6742-42a6-b859-8e11e17da788 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: c3f98fde-f0aa-415a-a8a4-02c829157a8f kind: Event lastTimestamp: "2026-04-24T22:49:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:32Z" name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn.18a96ca25fe58d06 namespace: kserve-ci-e2e-test resourceVersion: "18927" uid: 70dad5dc-d3e5-4b83-b84f-33807927e8f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: c3f98fde-f0aa-415a-a8a4-02c829157a8f kind: Event lastTimestamp: "2026-04-24T22:49:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:32Z" name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn.18a96ca260563e38 namespace: kserve-ci-e2e-test resourceVersion: "18928" uid: 89bdcd7f-584e-4e58-9be6-a485934bb85d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:49:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: c3f98fde-f0aa-415a-a8a4-02c829157a8f kind: Event lastTimestamp: "2026-04-24T22:50:10Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:49:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:10Z" name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn.18a96ca2d1c9b7f0 namespace: kserve-ci-e2e-test resourceVersion: "19237" uid: 96267ed5-ed47-4248-b681-86bc959bdcee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: c3f98fde-f0aa-415a-a8a4-02c829157a8f kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:42Z" name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn.18a96cb29ce8384a namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 8e567fca-4ee6-4f87-95d8-163c13ce1d39 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: c3f98fde-f0aa-415a-a8a4-02c829157a8f kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:42Z" name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn.18a96cb29ce91281 namespace: kserve-ci-e2e-test resourceVersion: "19781" uid: 4cb92f53-1b8e-4b71-adc0-199b25af49d2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: c3f98fde-f0aa-415a-a8a4-02c829157a8f kind: Event lastTimestamp: "2026-04-24T22:50:46Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-04-24T22:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:46Z" name: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn.18a96cb395a19535 namespace: kserve-ci-e2e-test resourceVersion: "19870" uid: a5a9c4e6-a4d1-49db-8909-dd7823cb7b06 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5c921-predictor-78cfcc75d6 namespace: kserve-ci-e2e-test resourceVersion: "18892" uid: 7585691b-2233-4978-8276-aca98430a90d kind: Event lastTimestamp: "2026-04-24T22:49:32Z" message: 'Created pod: error-404-isvc-5c921-predictor-78cfcc75d6-6fxpn' metadata: creationTimestamp: "2026-04-24T22:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:49:32Z" name: error-404-isvc-5c921-predictor-78cfcc75d6.18a96ca23d838287 namespace: kserve-ci-e2e-test resourceVersion: "18900" uid: 4d206150-a8e5-40f2-a751-30c4534b32b2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5c921-predictor namespace: kserve-ci-e2e-test resourceVersion: "18888" uid: e1039e69-678a-459d-84a4-c2de3aa089d9 kind: Event lastTimestamp: "2026-04-24T22:49:32Z" message: Scaled up replica set error-404-isvc-5c921-predictor-78cfcc75d6 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:49:32Z" name: error-404-isvc-5c921-predictor.18a96ca23c1c3109 namespace: kserve-ci-e2e-test resourceVersion: "18893" uid: 9a164755-6f24-4e18-9d76-ba31d7b260f7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:49:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5c921-predictor namespace: kserve-ci-e2e-test resourceVersion: "18891" uid: 81897f9f-a462-4a23-bb3d-56f63d239803 kind: Event lastTimestamp: "2026-04-24T22:50:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:02Z" name: error-404-isvc-5c921-predictor.18a96ca5bacb6174 namespace: kserve-ci-e2e-test resourceVersion: "19174" uid: df1012e8-fb41-4ca6-b48e-fabc572aca59 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:49:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5c921-predictor namespace: kserve-ci-e2e-test resourceVersion: "18891" uid: 81897f9f-a462-4a23-bb3d-56f63d239803 kind: Event lastTimestamp: "2026-04-24T22:50:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:49:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:02Z" name: error-404-isvc-5c921-predictor.18a96ca5bb2f6101 namespace: kserve-ci-e2e-test resourceVersion: "19175" uid: 09dd71d1-a341-4436-8fcf-c7dff1ddca41 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:50:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5c921-predictor namespace: kserve-ci-e2e-test resourceVersion: "19066" uid: 81897f9f-a462-4a23-bb3d-56f63d239803 kind: Event lastTimestamp: "2026-04-24T22:50:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:32Z" name: error-404-isvc-5c921-predictor.18a96cacb892a17c namespace: kserve-ci-e2e-test resourceVersion: "19594" uid: 31a6f816-4059-4735-84ba-12efd7da90df reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:50:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5c921-predictor namespace: kserve-ci-e2e-test resourceVersion: "19066" uid: 81897f9f-a462-4a23-bb3d-56f63d239803 kind: Event lastTimestamp: "2026-04-24T22:50:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:32Z" name: error-404-isvc-5c921-predictor.18a96cacb9040b10 namespace: kserve-ci-e2e-test resourceVersion: "19595" uid: 5cf5dbbf-ea6c-45f1-8bbc-6db63868d519 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:49:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5c921 namespace: kserve-ci-e2e-test resourceVersion: "18865" uid: 6485b82d-33d4-49fe-a2d9-6d7b39c3fa23 kind: Event lastTimestamp: "2026-04-24T22:50:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:50:20Z" name: error-404-isvc-5c921.18a96ca23c44d803 namespace: kserve-ci-e2e-test resourceVersion: "19481" uid: d468a2ee-1a88-461c-8a0c-f965bc80ee28 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5c921 namespace: kserve-ci-e2e-test resourceVersion: "18865" uid: 6485b82d-33d4-49fe-a2d9-6d7b39c3fa23 kind: Event lastTimestamp: "2026-04-24T22:49:32Z" message: 'Failed to update status for InferenceService "error-404-isvc-5c921": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5c921": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:49:32Z" name: error-404-isvc-5c921.18a96ca23cac7075 namespace: kserve-ci-e2e-test resourceVersion: "18896" uid: a2278a48-e4fd-4420-b8b0-45ef04a82d0f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5c921 namespace: kserve-ci-e2e-test resourceVersion: "18865" uid: 6485b82d-33d4-49fe-a2d9-6d7b39c3fa23 kind: Event lastTimestamp: "2026-04-24T22:49:32Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5c921": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:49:32Z" name: error-404-isvc-5c921.18a96ca23cacade6 namespace: kserve-ci-e2e-test resourceVersion: "18898" uid: 7659f3dc-ce5c-40a5-83be-83a663d40fb0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5c921 namespace: kserve-ci-e2e-test resourceVersion: "19477" uid: 6485b82d-33d4-49fe-a2d9-6d7b39c3fa23 kind: Event lastTimestamp: "2026-04-24T22:50:20Z" message: InferenceService [error-404-isvc-5c921] is Ready metadata: creationTimestamp: "2026-04-24T22:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:50:20Z" name: error-404-isvc-5c921.18a96cad8cb80eb9 namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: de88c8f5-95d4-497b-90f4-09f3e7beddad reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T23:01:07.663240Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq namespace: kserve-ci-e2e-test resourceVersion: "25052" uid: 76d12ff9-7ae6-4279-a316-130ac44adeca kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T23:01:07Z" name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq.18a96d44337a9291 namespace: kserve-ci-e2e-test resourceVersion: "25056" uid: 56d1df72-3f79-42ae-a217-85873982032d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:07Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq namespace: kserve-ci-e2e-test resourceVersion: "25054" uid: 76d12ff9-7ae6-4279-a316-130ac44adeca kind: Event lastTimestamp: "2026-04-24T23:01:07Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-be7a5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:07Z" name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq.18a96d443f4e852f namespace: kserve-ci-e2e-test resourceVersion: "25069" uid: 4a2775d8-db5b-4066-8533-33034c8da1b5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:08Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq namespace: kserve-ci-e2e-test resourceVersion: "25062" uid: 76d12ff9-7ae6-4279-a316-130ac44adeca kind: Event lastTimestamp: "2026-04-24T23:01:08Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T23:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T23:01:08Z" name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq.18a96d4470f7f6bc namespace: kserve-ci-e2e-test resourceVersion: "25088" uid: c1b8867a-57e8-420b-adca-7c054ebe7564 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq namespace: kserve-ci-e2e-test resourceVersion: "25054" uid: 76d12ff9-7ae6-4279-a316-130ac44adeca kind: Event lastTimestamp: "2026-04-24T23:01:08Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T23:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:08Z" name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq.18a96d447267cc31 namespace: kserve-ci-e2e-test resourceVersion: "25092" uid: f160f7d5-418c-4315-803a-5b4c9357de5b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq namespace: kserve-ci-e2e-test resourceVersion: "25054" uid: 76d12ff9-7ae6-4279-a316-130ac44adeca kind: Event lastTimestamp: "2026-04-24T23:01:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T23:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:08Z" name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq.18a96d4475c7035d namespace: kserve-ci-e2e-test resourceVersion: "25099" uid: ad48f9d7-1618-49e4-95f1-a7d021aadd99 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq namespace: kserve-ci-e2e-test resourceVersion: "25054" uid: 76d12ff9-7ae6-4279-a316-130ac44adeca kind: Event lastTimestamp: "2026-04-24T23:01:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T23:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:08Z" name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq.18a96d44764c9c98 namespace: kserve-ci-e2e-test resourceVersion: "25100" uid: 8857cd47-7fcd-4b3a-b6af-1936988e5fd9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq namespace: kserve-ci-e2e-test resourceVersion: "25054" uid: 76d12ff9-7ae6-4279-a316-130ac44adeca kind: Event lastTimestamp: "2026-04-24T23:01:08Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T23:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:08Z" name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq.18a96d4476580b21 namespace: kserve-ci-e2e-test resourceVersion: "25101" uid: baa97b0a-b5ae-41f2-a517-38005892d408 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq namespace: kserve-ci-e2e-test resourceVersion: "25054" uid: 76d12ff9-7ae6-4279-a316-130ac44adeca kind: Event lastTimestamp: "2026-04-24T23:01:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T23:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:08Z" name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq.18a96d447947943b namespace: kserve-ci-e2e-test resourceVersion: "25102" uid: 3ae23274-86c2-45ae-85dd-7ceaea18d74e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq namespace: kserve-ci-e2e-test resourceVersion: "25054" uid: 76d12ff9-7ae6-4279-a316-130ac44adeca kind: Event lastTimestamp: "2026-04-24T23:01:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T23:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:08Z" name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq.18a96d4479be1b01 namespace: kserve-ci-e2e-test resourceVersion: "25103" uid: d435c45a-9612-4463-979a-ff018d76dcc7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T23:01:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq namespace: kserve-ci-e2e-test resourceVersion: "25054" uid: 76d12ff9-7ae6-4279-a316-130ac44adeca kind: Event lastTimestamp: "2026-04-24T23:01:45Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T23:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:45Z" name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq.18a96d44bdfc43bf namespace: kserve-ci-e2e-test resourceVersion: "25468" uid: 93361ae9-0c2a-4da1-a9f2-58b83df3755a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:10:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq namespace: kserve-ci-e2e-test resourceVersion: "25054" uid: 76d12ff9-7ae6-4279-a316-130ac44adeca kind: Event lastTimestamp: "2026-04-24T23:10:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T23:10:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:10:22Z" name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq.18a96dc55b1f24e4 namespace: kserve-ci-e2e-test resourceVersion: "29438" uid: 82f60895-b51e-44fa-abd5-c6d980d8f248 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:10:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq namespace: kserve-ci-e2e-test resourceVersion: "25054" uid: 76d12ff9-7ae6-4279-a316-130ac44adeca kind: Event lastTimestamp: "2026-04-24T23:10:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T23:10:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:10:22Z" name: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq.18a96dc55b1fbc6b namespace: kserve-ci-e2e-test resourceVersion: "29439" uid: fcf457f3-9aa0-469c-93ae-ec02787518fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-be7a5-predictor-5c7766ddbb namespace: kserve-ci-e2e-test resourceVersion: "25045" uid: 845ea3c6-e4d4-47ba-a587-12b1f7d7373a kind: Event lastTimestamp: "2026-04-24T23:01:07Z" message: 'Created pod: error-404-isvc-be7a5-predictor-5c7766ddbb-8xswq' metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:01:07Z" name: error-404-isvc-be7a5-predictor-5c7766ddbb.18a96d44330f8394 namespace: kserve-ci-e2e-test resourceVersion: "25053" uid: ed5cb376-6cf2-4167-94bd-3b50c21ab116 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-be7a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "25043" uid: 85b181d8-2944-4833-b67b-b85a3ab9b05c kind: Event lastTimestamp: "2026-04-24T23:01:07Z" message: Scaled up replica set error-404-isvc-be7a5-predictor-5c7766ddbb from 0 to 1 metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:01:07Z" name: error-404-isvc-be7a5-predictor.18a96d44316300f2 namespace: kserve-ci-e2e-test resourceVersion: "25047" uid: 742b2bd1-d615-46a8-9737-a2b44ea4554d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T23:01:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-be7a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "25044" uid: 5a616a46-1711-44d2-b0b8-1e3858bce0da kind: Event lastTimestamp: "2026-04-24T23:01:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T23:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:01:52Z" name: error-404-isvc-be7a5-predictor.18a96d47afde57f8 namespace: kserve-ci-e2e-test resourceVersion: "25525" uid: 100b6b80-239c-4d62-a8cb-6ae1526de04f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T23:01:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-be7a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "25044" uid: 5a616a46-1711-44d2-b0b8-1e3858bce0da kind: Event lastTimestamp: "2026-04-24T23:01:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T23:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:01:52Z" name: error-404-isvc-be7a5-predictor.18a96d47b0416be5 namespace: kserve-ci-e2e-test resourceVersion: "25526" uid: 324dacd6-ef14-4761-ba89-fb3e8aa23534 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T23:02:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-be7a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "25230" uid: 5a616a46-1711-44d2-b0b8-1e3858bce0da kind: Event lastTimestamp: "2026-04-24T23:02:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T23:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:02:37Z" name: error-404-isvc-be7a5-predictor.18a96d522c48533a namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: f155172c-a7fd-41a8-9f4d-0b4ba98269a4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T23:02:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-be7a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "25230" uid: 5a616a46-1711-44d2-b0b8-1e3858bce0da kind: Event lastTimestamp: "2026-04-24T23:02:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T23:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:02:37Z" name: error-404-isvc-be7a5-predictor.18a96d522c8e4b84 namespace: kserve-ci-e2e-test resourceVersion: "25911" uid: b0c794a9-6e27-4b91-84d7-423761495c72 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T23:01:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-be7a5 namespace: kserve-ci-e2e-test resourceVersion: "25018" uid: de87cc2c-0eae-4ae6-8f9f-40e64779e1a8 kind: Event lastTimestamp: "2026-04-24T23:01:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:01:56Z" name: error-404-isvc-be7a5.18a96d44313c3ac3 namespace: kserve-ci-e2e-test resourceVersion: "25567" uid: a6f76867-0480-445c-b0f6-eb5a9bf9c00e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-be7a5 namespace: kserve-ci-e2e-test resourceVersion: "25018" uid: de87cc2c-0eae-4ae6-8f9f-40e64779e1a8 kind: Event lastTimestamp: "2026-04-24T23:01:07Z" message: 'Failed to update status for InferenceService "error-404-isvc-be7a5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-be7a5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:01:07Z" name: error-404-isvc-be7a5.18a96d44319300b7 namespace: kserve-ci-e2e-test resourceVersion: "25049" uid: c45b41f3-6e86-4f5c-af4a-71126d3c115c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-be7a5 namespace: kserve-ci-e2e-test resourceVersion: "25018" uid: de87cc2c-0eae-4ae6-8f9f-40e64779e1a8 kind: Event lastTimestamp: "2026-04-24T23:01:07Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-be7a5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:01:07Z" name: error-404-isvc-be7a5.18a96d4431936d5d namespace: kserve-ci-e2e-test resourceVersion: "25050" uid: a4b60e2f-605b-4065-8ad1-127f4a367d3a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-be7a5 namespace: kserve-ci-e2e-test resourceVersion: "25565" uid: de87cc2c-0eae-4ae6-8f9f-40e64779e1a8 kind: Event lastTimestamp: "2026-04-24T23:01:56Z" message: InferenceService [error-404-isvc-be7a5] is Ready metadata: creationTimestamp: "2026-04-24T23:01:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:01:56Z" name: error-404-isvc-be7a5.18a96d4f78ddca58 namespace: kserve-ci-e2e-test resourceVersion: "25566" uid: 1a7bcf21-bb2e-417e-b9ae-c09cd95b78b6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T23:00:37.218819Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz namespace: kserve-ci-e2e-test resourceVersion: "24603" uid: 11a841c7-77cf-462a-a9f8-553c22eb81d7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d0532-predictor-7bbf598d6b-xswpz to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T23:00:37Z" name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz.18a96d3d1cd99711 namespace: kserve-ci-e2e-test resourceVersion: "24607" uid: 7bb48a78-fb56-428b-82f0-cd8685e21a8c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz namespace: kserve-ci-e2e-test resourceVersion: "24613" uid: 11a841c7-77cf-462a-a9f8-553c22eb81d7 kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T23:00:37Z" name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz.18a96d3d3647aa08 namespace: kserve-ci-e2e-test resourceVersion: "24629" uid: 9e06b20e-bc8a-4c1c-adec-358e3ccea366 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz namespace: kserve-ci-e2e-test resourceVersion: "24605" uid: 11a841c7-77cf-462a-a9f8-553c22eb81d7 kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:37Z" name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz.18a96d3d37961a57 namespace: kserve-ci-e2e-test resourceVersion: "24631" uid: d8622136-e3d6-4214-938b-4ecc46270b3c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz namespace: kserve-ci-e2e-test resourceVersion: "24605" uid: 11a841c7-77cf-462a-a9f8-553c22eb81d7 kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:37Z" name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz.18a96d3d3a67525f namespace: kserve-ci-e2e-test resourceVersion: "24634" uid: 1365c3cb-9166-418d-87e0-ae4748499efb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz namespace: kserve-ci-e2e-test resourceVersion: "24605" uid: 11a841c7-77cf-462a-a9f8-553c22eb81d7 kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:37Z" name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz.18a96d3d3ad8b35e namespace: kserve-ci-e2e-test resourceVersion: "24635" uid: ac3a5bd9-a465-4fc1-8d36-75bd30f4fb12 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz namespace: kserve-ci-e2e-test resourceVersion: "24605" uid: 11a841c7-77cf-462a-a9f8-553c22eb81d7 kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:37Z" name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz.18a96d3d3ae53d99 namespace: kserve-ci-e2e-test resourceVersion: "24636" uid: 4c7bee78-01d8-490a-a1cb-cd1b900906a4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz namespace: kserve-ci-e2e-test resourceVersion: "24605" uid: 11a841c7-77cf-462a-a9f8-553c22eb81d7 kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:37Z" name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz.18a96d3d3eb2174e namespace: kserve-ci-e2e-test resourceVersion: "24637" uid: 3695d6b7-adb1-413e-b9a0-164a38cc417c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz namespace: kserve-ci-e2e-test resourceVersion: "24605" uid: 11a841c7-77cf-462a-a9f8-553c22eb81d7 kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:37Z" name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz.18a96d3d3f276666 namespace: kserve-ci-e2e-test resourceVersion: "24638" uid: d8580031-6079-4577-8b28-1c5ed324eac9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T23:00:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz namespace: kserve-ci-e2e-test resourceVersion: "24605" uid: 11a841c7-77cf-462a-a9f8-553c22eb81d7 kind: Event lastTimestamp: "2026-04-24T23:01:14Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T23:00:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:14Z" name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz.18a96d3d7f480f41 namespace: kserve-ci-e2e-test resourceVersion: "25160" uid: 17403715-dc06-4cc5-be39-ded32c03b5ca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:17:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz namespace: kserve-ci-e2e-test resourceVersion: "24605" uid: 11a841c7-77cf-462a-a9f8-553c22eb81d7 kind: Event lastTimestamp: "2026-04-24T23:17:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T23:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:17:56Z" name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz.18a96e2f2babd8e0 namespace: kserve-ci-e2e-test resourceVersion: "32597" uid: 83d931c1-bd05-4dce-b6c1-30cf4dfae6e5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:17:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz namespace: kserve-ci-e2e-test resourceVersion: "24605" uid: 11a841c7-77cf-462a-a9f8-553c22eb81d7 kind: Event lastTimestamp: "2026-04-24T23:17:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T23:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:17:56Z" name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz.18a96e2f2bace8f1 namespace: kserve-ci-e2e-test resourceVersion: "32598" uid: 0f4cc10f-55a3-4f66-8c69-9f011bd35572 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:18:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz namespace: kserve-ci-e2e-test resourceVersion: "24605" uid: 11a841c7-77cf-462a-a9f8-553c22eb81d7 kind: Event lastTimestamp: "2026-04-24T23:18:00Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-04-24T23:18:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:18:00Z" name: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz.18a96e301b7de093 namespace: kserve-ci-e2e-test resourceVersion: "32666" uid: 86d04fba-a289-439f-b7ea-959dad6f2616 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d0532-predictor-7bbf598d6b namespace: kserve-ci-e2e-test resourceVersion: "24596" uid: 51dfe935-cea8-4e26-a3c1-7818491a7120 kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: 'Created pod: error-404-isvc-d0532-predictor-7bbf598d6b-xswpz' metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:00:37Z" name: error-404-isvc-d0532-predictor-7bbf598d6b.18a96d3d1c605c0f namespace: kserve-ci-e2e-test resourceVersion: "24604" uid: 9267c4f9-d649-4d8d-96a5-2e4260dec4b9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d0532-predictor namespace: kserve-ci-e2e-test resourceVersion: "24593" uid: a56d28fd-2f03-4680-b7f5-bab7f5c9c45f kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: Scaled up replica set error-404-isvc-d0532-predictor-7bbf598d6b from 0 to 1 metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:00:37Z" name: error-404-isvc-d0532-predictor.18a96d3d1acf6a99 namespace: kserve-ci-e2e-test resourceVersion: "24598" uid: 39fd4829-9c30-4556-b660-f4358f586f69 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T23:00:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d0532-predictor namespace: kserve-ci-e2e-test resourceVersion: "24595" uid: 92e5090b-85f1-4297-a1ff-28ddf13fb2cb kind: Event lastTimestamp: "2026-04-24T23:01:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T23:00:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:01:07Z" name: error-404-isvc-d0532-predictor.18a96d40993f2ad6 namespace: kserve-ci-e2e-test resourceVersion: "24954" uid: 218e2048-13f0-43e8-a432-7dcc0f2f6be3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T23:00:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d0532-predictor namespace: kserve-ci-e2e-test resourceVersion: "24595" uid: 92e5090b-85f1-4297-a1ff-28ddf13fb2cb kind: Event lastTimestamp: "2026-04-24T23:01:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T23:00:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:01:07Z" name: error-404-isvc-d0532-predictor.18a96d4099a2c15a namespace: kserve-ci-e2e-test resourceVersion: "24956" uid: 7ed0de95-8890-49d0-962d-a7284eed61db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T23:01:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d0532-predictor namespace: kserve-ci-e2e-test resourceVersion: "24792" uid: 92e5090b-85f1-4297-a1ff-28ddf13fb2cb kind: Event lastTimestamp: "2026-04-24T23:02:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T23:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:02:07Z" name: error-404-isvc-d0532-predictor.18a96d4797a9c521 namespace: kserve-ci-e2e-test resourceVersion: "25645" uid: f91e83a1-ad2e-4fd1-97a1-03af1fc56a49 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T23:01:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d0532-predictor namespace: kserve-ci-e2e-test resourceVersion: "24792" uid: 92e5090b-85f1-4297-a1ff-28ddf13fb2cb kind: Event lastTimestamp: "2026-04-24T23:02:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T23:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:02:07Z" name: error-404-isvc-d0532-predictor.18a96d4798c3794a namespace: kserve-ci-e2e-test resourceVersion: "25646" uid: 0a815f9e-a50d-4da5-8d61-858039334674 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d0532 namespace: kserve-ci-e2e-test resourceVersion: "24562" uid: 5b7c7de7-5aec-49b0-93a5-ae1d3c21e875 kind: Event lastTimestamp: "2026-04-24T23:01:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:01:24Z" name: error-404-isvc-d0532.18a96d3d1aa6c97c namespace: kserve-ci-e2e-test resourceVersion: "25253" uid: 6b3e5685-e185-4538-aef1-f901f8e835cf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d0532 namespace: kserve-ci-e2e-test resourceVersion: "24562" uid: 5b7c7de7-5aec-49b0-93a5-ae1d3c21e875 kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: 'Failed to update status for InferenceService "error-404-isvc-d0532": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d0532": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:00:37Z" name: error-404-isvc-d0532.18a96d3d1b1363e8 namespace: kserve-ci-e2e-test resourceVersion: "24600" uid: 713b593a-7275-41ed-8c09-07d95898f130 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d0532 namespace: kserve-ci-e2e-test resourceVersion: "24562" uid: 5b7c7de7-5aec-49b0-93a5-ae1d3c21e875 kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d0532": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:00:37Z" name: error-404-isvc-d0532.18a96d3d1b13c761 namespace: kserve-ci-e2e-test resourceVersion: "24601" uid: 9a9c4045-e01f-4127-9698-116944b55502 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d0532 namespace: kserve-ci-e2e-test resourceVersion: "25251" uid: 5b7c7de7-5aec-49b0-93a5-ae1d3c21e875 kind: Event lastTimestamp: "2026-04-24T23:01:24Z" message: InferenceService [error-404-isvc-d0532] is Ready metadata: creationTimestamp: "2026-04-24T23:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:01:24Z" name: error-404-isvc-d0532.18a96d483a5fcae8 namespace: kserve-ci-e2e-test resourceVersion: "25252" uid: f1de49df-733e-45d4-85fa-a8c225817c64 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:40:17.345424Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e938d-predictor-8478f5b96-brdgr namespace: kserve-ci-e2e-test resourceVersion: "14504" uid: 68aef45b-ce2a-449b-9034-2c357befa414 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e938d-predictor-8478f5b96-brdgr to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-04-24T22:40:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:40:17Z" name: error-404-isvc-e938d-predictor-8478f5b96-brdgr.18a96c2116bac3db namespace: kserve-ci-e2e-test resourceVersion: "14509" uid: 31570f87-33ed-4996-b8af-7fec983fe010 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e938d-predictor-8478f5b96-brdgr namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: 68aef45b-ce2a-449b-9034-2c357befa414 kind: Event lastTimestamp: "2026-04-24T22:40:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-e938d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T22:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:17Z" name: error-404-isvc-e938d-predictor-8478f5b96-brdgr.18a96c21257f4a1e namespace: kserve-ci-e2e-test resourceVersion: "14520" uid: a8746a28-f4c6-44fd-90c9-03027d5baf3e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:18Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e938d-predictor-8478f5b96-brdgr namespace: kserve-ci-e2e-test resourceVersion: "14512" uid: 68aef45b-ce2a-449b-9034-2c357befa414 kind: Event lastTimestamp: "2026-04-24T22:40:18Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:40:18Z" name: error-404-isvc-e938d-predictor-8478f5b96-brdgr.18a96c215575ab04 namespace: kserve-ci-e2e-test resourceVersion: "14542" uid: 8e628c84-02fa-409c-8cf0-7764200dc0f0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e938d-predictor-8478f5b96-brdgr namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: 68aef45b-ce2a-449b-9034-2c357befa414 kind: Event lastTimestamp: "2026-04-24T22:40:18Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:18Z" name: error-404-isvc-e938d-predictor-8478f5b96-brdgr.18a96c2156d576d6 namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: 2461a13d-4112-451e-a294-e9e7feec6e62 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e938d-predictor-8478f5b96-brdgr namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: 68aef45b-ce2a-449b-9034-2c357befa414 kind: Event lastTimestamp: "2026-04-24T22:40:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:18Z" name: error-404-isvc-e938d-predictor-8478f5b96-brdgr.18a96c215a53a4ff namespace: kserve-ci-e2e-test resourceVersion: "14552" uid: 21392492-bf38-4263-82e8-e633d86a235f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e938d-predictor-8478f5b96-brdgr namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: 68aef45b-ce2a-449b-9034-2c357befa414 kind: Event lastTimestamp: "2026-04-24T22:40:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:18Z" name: error-404-isvc-e938d-predictor-8478f5b96-brdgr.18a96c215aca24fb namespace: kserve-ci-e2e-test resourceVersion: "14553" uid: 39c3f870-5394-4399-ae8b-24cf1f62ca79 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e938d-predictor-8478f5b96-brdgr namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: 68aef45b-ce2a-449b-9034-2c357befa414 kind: Event lastTimestamp: "2026-04-24T22:40:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:18Z" name: error-404-isvc-e938d-predictor-8478f5b96-brdgr.18a96c215ad76346 namespace: kserve-ci-e2e-test resourceVersion: "14554" uid: 3c5e1209-7c96-430d-a98e-e91dabeae902 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e938d-predictor-8478f5b96-brdgr namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: 68aef45b-ce2a-449b-9034-2c357befa414 kind: Event lastTimestamp: "2026-04-24T22:40:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:18Z" name: error-404-isvc-e938d-predictor-8478f5b96-brdgr.18a96c215e1c9643 namespace: kserve-ci-e2e-test resourceVersion: "14555" uid: ad6bf46a-89a0-476b-876e-ccd41a376fda reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e938d-predictor-8478f5b96-brdgr namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: 68aef45b-ce2a-449b-9034-2c357befa414 kind: Event lastTimestamp: "2026-04-24T22:40:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:18Z" name: error-404-isvc-e938d-predictor-8478f5b96-brdgr.18a96c215e9b187f namespace: kserve-ci-e2e-test resourceVersion: "14556" uid: 514e18f0-6ff4-4f17-aead-6dcc04286d0f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T22:40:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e938d-predictor-8478f5b96-brdgr namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: 68aef45b-ce2a-449b-9034-2c357befa414 kind: Event lastTimestamp: "2026-04-24T22:49:34Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:34Z" name: error-404-isvc-e938d-predictor-8478f5b96-brdgr.18a96c216352b405 namespace: kserve-ci-e2e-test resourceVersion: "18960" uid: 6e08af77-ffff-439e-9199-fe2a4c6fe9b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e938d-predictor-8478f5b96-brdgr namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: 68aef45b-ce2a-449b-9034-2c357befa414 kind: Event lastTimestamp: "2026-04-24T22:49:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:31Z" name: error-404-isvc-e938d-predictor-8478f5b96-brdgr.18a96ca2393353c8 namespace: kserve-ci-e2e-test resourceVersion: "18872" uid: 48938f25-f292-48fe-bebb-52cc75860542 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e938d-predictor-8478f5b96-brdgr namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: 68aef45b-ce2a-449b-9034-2c357befa414 kind: Event lastTimestamp: "2026-04-24T22:49:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:31Z" name: error-404-isvc-e938d-predictor-8478f5b96-brdgr.18a96ca23933fd79 namespace: kserve-ci-e2e-test resourceVersion: "18876" uid: b25bc8dc-819a-474c-bea7-5955cf32d787 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e938d-predictor-8478f5b96-brdgr namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: 68aef45b-ce2a-449b-9034-2c357befa414 kind: Event lastTimestamp: "2026-04-24T22:49:34Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:49:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:34Z" name: error-404-isvc-e938d-predictor-8478f5b96-brdgr.18a96ca2d7aadc64 namespace: kserve-ci-e2e-test resourceVersion: "18959" uid: 3267db4d-3be0-4450-a2ee-14bdd430aef6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e938d-predictor-8478f5b96 namespace: kserve-ci-e2e-test resourceVersion: "14498" uid: 37a67972-625a-455e-bd53-b640e63ac006 kind: Event lastTimestamp: "2026-04-24T22:40:17Z" message: 'Created pod: error-404-isvc-e938d-predictor-8478f5b96-brdgr' metadata: creationTimestamp: "2026-04-24T22:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:17Z" name: error-404-isvc-e938d-predictor-8478f5b96.18a96c211671ba1f namespace: kserve-ci-e2e-test resourceVersion: "14505" uid: 04b488ba-14e3-47c7-8c8b-51709723f732 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e938d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14496" uid: 98143ed7-fa63-4e27-9616-f9b57a5f4acc kind: Event lastTimestamp: "2026-04-24T22:40:17Z" message: Scaled up replica set error-404-isvc-e938d-predictor-8478f5b96 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:17Z" name: error-404-isvc-e938d-predictor.18a96c211490514f namespace: kserve-ci-e2e-test resourceVersion: "14500" uid: d872e3fb-dc19-472d-b940-e5ab1cfd6b56 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:40:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e938d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: f942cbbc-cf7f-4a6b-af03-29832bd15020 kind: Event lastTimestamp: "2026-04-24T22:40:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:47Z" name: error-404-isvc-e938d-predictor.18a96c24931cc51e namespace: kserve-ci-e2e-test resourceVersion: "14851" uid: 85da45b5-d46b-4178-9fd7-a3a4ac9e9820 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:40:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e938d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: f942cbbc-cf7f-4a6b-af03-29832bd15020 kind: Event lastTimestamp: "2026-04-24T22:40:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:47Z" name: error-404-isvc-e938d-predictor.18a96c249382d91c namespace: kserve-ci-e2e-test resourceVersion: "14852" uid: 7e98e063-82b5-4faa-b27b-02218f9237ea reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:41:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e938d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14696" uid: f942cbbc-cf7f-4a6b-af03-29832bd15020 kind: Event lastTimestamp: "2026-04-24T22:41:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:41:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:32Z" name: error-404-isvc-e938d-predictor.18a96c2b90e6e890 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 6580c638-364d-463f-9ada-4c229188e0ad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:41:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e938d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14696" uid: f942cbbc-cf7f-4a6b-af03-29832bd15020 kind: Event lastTimestamp: "2026-04-24T22:41:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:41:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:32Z" name: error-404-isvc-e938d-predictor.18a96c2b914d35be namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: bf181fca-2ded-44d6-871f-9b3e945534f6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:40:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e938d namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 2c115a46-01cb-4607-a533-0edff0ab59ad kind: Event lastTimestamp: "2026-04-24T22:41:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:41:04Z" name: error-404-isvc-e938d.18a96c2114700212 namespace: kserve-ci-e2e-test resourceVersion: "15178" uid: c9ada1c8-7e92-420d-8062-497a2761fa67 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e938d namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 2c115a46-01cb-4607-a533-0edff0ab59ad kind: Event lastTimestamp: "2026-04-24T22:40:17Z" message: 'Failed to update status for InferenceService "error-404-isvc-e938d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e938d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:40:17Z" name: error-404-isvc-e938d.18a96c21152996d4 namespace: kserve-ci-e2e-test resourceVersion: "14502" uid: 0ab65979-0310-4ba9-b2e9-797b1d160a94 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e938d namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 2c115a46-01cb-4607-a533-0edff0ab59ad kind: Event lastTimestamp: "2026-04-24T22:40:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e938d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:40:17Z" name: error-404-isvc-e938d.18a96c211529ec65 namespace: kserve-ci-e2e-test resourceVersion: "14508" uid: 19ba1227-7cc5-456f-8efa-7fa223f25fe7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e938d namespace: kserve-ci-e2e-test resourceVersion: "15176" uid: 2c115a46-01cb-4607-a533-0edff0ab59ad kind: Event lastTimestamp: "2026-04-24T22:41:04Z" message: InferenceService [error-404-isvc-e938d] is Ready metadata: creationTimestamp: "2026-04-24T22:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:41:04Z" name: error-404-isvc-e938d.18a96c2c1eab57e2 namespace: kserve-ci-e2e-test resourceVersion: "15177" uid: cdbb52c9-1908-458e-91a7-f2b0d7ea2ebd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:40:57.099491Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt namespace: kserve-ci-e2e-test resourceVersion: "15052" uid: 77c00532-53e1-4786-8092-b743bfe09f10 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f8431-predictor-6b847f4b94-q6dxt to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-04-24T22:40:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:40:57Z" name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt.18a96c2a5841b290 namespace: kserve-ci-e2e-test resourceVersion: "15056" uid: f6f70704-e4ca-4bde-8799-04b4a1334eb0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: 77c00532-53e1-4786-8092-b743bfe09f10 kind: Event lastTimestamp: "2026-04-24T22:40:57Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-f8431-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T22:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:57Z" name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt.18a96c2a666dfd54 namespace: kserve-ci-e2e-test resourceVersion: "15064" uid: 5e863ffc-b383-448a-a119-339ef00de593 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:58Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt namespace: kserve-ci-e2e-test resourceVersion: "15062" uid: 77c00532-53e1-4786-8092-b743bfe09f10 kind: Event lastTimestamp: "2026-04-24T22:40:58Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:40:58Z" name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt.18a96c2a957409d1 namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: a7da97fa-9e1a-4394-94ae-ae7c83e61be4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: 77c00532-53e1-4786-8092-b743bfe09f10 kind: Event lastTimestamp: "2026-04-24T22:40:58Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:58Z" name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt.18a96c2a96f65521 namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: 62b48a4f-f0b9-45ce-9506-b248d4874ca8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: 77c00532-53e1-4786-8092-b743bfe09f10 kind: Event lastTimestamp: "2026-04-24T22:40:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:58Z" name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt.18a96c2a99ead81f namespace: kserve-ci-e2e-test resourceVersion: "15089" uid: e484f6f7-c9f8-42cc-9d14-6eb0f05b9ba3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: 77c00532-53e1-4786-8092-b743bfe09f10 kind: Event lastTimestamp: "2026-04-24T22:40:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:58Z" name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt.18a96c2a9a5f244b namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: fde07327-73ec-4c29-92b8-9102ea737ef0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: 77c00532-53e1-4786-8092-b743bfe09f10 kind: Event lastTimestamp: "2026-04-24T22:40:58Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:58Z" name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt.18a96c2a9a6b7ecc namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: 6b420f2c-4605-4cc6-9808-3c8cd33f995c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: 77c00532-53e1-4786-8092-b743bfe09f10 kind: Event lastTimestamp: "2026-04-24T22:40:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:58Z" name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt.18a96c2a9d8109fc namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: 08090037-27ab-453e-88fb-40762abbfad9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: 77c00532-53e1-4786-8092-b743bfe09f10 kind: Event lastTimestamp: "2026-04-24T22:40:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:58Z" name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt.18a96c2a9df8107e namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: 0456f59e-34cc-4f4e-be29-097a85d46cc8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:40:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: 77c00532-53e1-4786-8092-b743bfe09f10 kind: Event lastTimestamp: "2026-04-24T22:41:35Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:41:35Z" name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt.18a96c2af7a14609 namespace: kserve-ci-e2e-test resourceVersion: "15483" uid: fcd24602-c6b3-41c6-b119-e04294376056 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: 77c00532-53e1-4786-8092-b743bfe09f10 kind: Event lastTimestamp: "2026-04-24T22:50:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:12Z" name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt.18a96cab8af8718a namespace: kserve-ci-e2e-test resourceVersion: "19333" uid: d788c555-6236-42e7-b838-4a661376b592 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: 77c00532-53e1-4786-8092-b743bfe09f10 kind: Event lastTimestamp: "2026-04-24T22:50:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:12Z" name: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt.18a96cab8af9a935 namespace: kserve-ci-e2e-test resourceVersion: "19335" uid: a775742f-5548-4a02-8c9a-23fff2b3665b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f8431-predictor-6b847f4b94 namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: a912297b-2295-4416-b7a3-e82879c81129 kind: Event lastTimestamp: "2026-04-24T22:40:57Z" message: 'Created pod: error-404-isvc-f8431-predictor-6b847f4b94-q6dxt' metadata: creationTimestamp: "2026-04-24T22:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:57Z" name: error-404-isvc-f8431-predictor-6b847f4b94.18a96c2a57b69479 namespace: kserve-ci-e2e-test resourceVersion: "15053" uid: 8be8a8e1-6a9c-4c78-9bb0-5676c5cd6e05 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f8431-predictor namespace: kserve-ci-e2e-test resourceVersion: "15042" uid: 8f737693-52aa-4202-9778-84a871e4c02f kind: Event lastTimestamp: "2026-04-24T22:40:57Z" message: Scaled up replica set error-404-isvc-f8431-predictor-6b847f4b94 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:57Z" name: error-404-isvc-f8431-predictor.18a96c2a55dc93cf namespace: kserve-ci-e2e-test resourceVersion: "15047" uid: 848881b4-64ba-4a63-b994-bb63d0e9f29e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:41:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f8431-predictor namespace: kserve-ci-e2e-test resourceVersion: "15043" uid: e51a46be-8ed7-406e-b821-fdc8d2f43895 kind: Event lastTimestamp: "2026-04-24T22:41:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:41:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:27Z" name: error-404-isvc-f8431-predictor.18a96c2dd458782d namespace: kserve-ci-e2e-test resourceVersion: "15412" uid: e591a7c9-d475-4e13-a70c-80beffc4cd1d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:41:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f8431-predictor namespace: kserve-ci-e2e-test resourceVersion: "15043" uid: e51a46be-8ed7-406e-b821-fdc8d2f43895 kind: Event lastTimestamp: "2026-04-24T22:41:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:41:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:27Z" name: error-404-isvc-f8431-predictor.18a96c2dd4c4c3ec namespace: kserve-ci-e2e-test resourceVersion: "15413" uid: 76500b3f-d3e0-4fd9-bba1-fda374387274 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:41:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f8431-predictor namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: e51a46be-8ed7-406e-b821-fdc8d2f43895 kind: Event lastTimestamp: "2026-04-24T22:41:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:57Z" name: error-404-isvc-f8431-predictor.18a96c34d24a451c namespace: kserve-ci-e2e-test resourceVersion: "15692" uid: 81337830-3da0-4ccb-adf2-e01908017ff1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:41:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f8431-predictor namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: e51a46be-8ed7-406e-b821-fdc8d2f43895 kind: Event lastTimestamp: "2026-04-24T22:41:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:57Z" name: error-404-isvc-f8431-predictor.18a96c34d2b0581d namespace: kserve-ci-e2e-test resourceVersion: "15693" uid: f1d91c37-fb4b-44f7-b071-1d649691afe3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:40:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f8431 namespace: kserve-ci-e2e-test resourceVersion: "15010" uid: b62c3fac-80c4-425f-843c-6d8da2f2ab1c kind: Event lastTimestamp: "2026-04-24T22:41:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:41:45Z" name: error-404-isvc-f8431.18a96c2a55b12f6e namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: 2c787abe-1388-488c-b69a-da6f80412420 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f8431 namespace: kserve-ci-e2e-test resourceVersion: "15010" uid: b62c3fac-80c4-425f-843c-6d8da2f2ab1c kind: Event lastTimestamp: "2026-04-24T22:40:57Z" message: 'Failed to update status for InferenceService "error-404-isvc-f8431": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f8431": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:40:57Z" name: error-404-isvc-f8431.18a96c2a5636f607 namespace: kserve-ci-e2e-test resourceVersion: "15049" uid: 6c723c82-a606-4c4c-8c61-7b13764fe1c6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f8431 namespace: kserve-ci-e2e-test resourceVersion: "15010" uid: b62c3fac-80c4-425f-843c-6d8da2f2ab1c kind: Event lastTimestamp: "2026-04-24T22:40:57Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f8431": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:40:57Z" name: error-404-isvc-f8431.18a96c2a56372bef namespace: kserve-ci-e2e-test resourceVersion: "15050" uid: 0f31ee5b-b6e0-43c1-95c1-3d0a2c581b36 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f8431 namespace: kserve-ci-e2e-test resourceVersion: "15577" uid: b62c3fac-80c4-425f-843c-6d8da2f2ab1c kind: Event lastTimestamp: "2026-04-24T22:41:45Z" message: InferenceService [error-404-isvc-f8431] is Ready metadata: creationTimestamp: "2026-04-24T22:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:41:45Z" name: error-404-isvc-f8431.18a96c35b265534e namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: eb26a049-8c30-4ceb-94cf-3d9dac1e894a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:38:46.781750Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m namespace: kserve-ci-e2e-test resourceVersion: "13369" uid: 4d081b6b-d731-4112-85d1-cd2fe0fe77da kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m to ip-10-0-133-161.ec2.internal metadata: creationTimestamp: "2026-04-24T22:38:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:38:46Z" name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m.18a96c0c00b6be78 namespace: kserve-ci-e2e-test resourceVersion: "13376" uid: fa6516ee-9c7a-4c6b-a88b-2521b5554a2a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:47Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m namespace: kserve-ci-e2e-test resourceVersion: "13382" uid: 4d081b6b-d731-4112-85d1-cd2fe0fe77da kind: Event lastTimestamp: "2026-04-24T22:38:47Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:38:47Z" name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m.18a96c0c1a1820bd namespace: kserve-ci-e2e-test resourceVersion: "13453" uid: 2244bf5a-d261-4a5e-afe3-10b46309a628 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:47Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: 4d081b6b-d731-4112-85d1-cd2fe0fe77da kind: Event lastTimestamp: "2026-04-24T22:38:47Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:4ae4df75945dd799b0c11b5e37a48a5ba8230ff3507d345ea07c8897a8c39885" metadata: creationTimestamp: "2026-04-24T22:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:47Z" name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m.18a96c0c1bacf1ae namespace: kserve-ci-e2e-test resourceVersion: "13455" uid: 4eddc82d-76f0-4b9f-bd6c-d7c8b48c09c9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: 4d081b6b-d731-4112-85d1-cd2fe0fe77da kind: Event lastTimestamp: "2026-04-24T22:38:50Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:4ae4df75945dd799b0c11b5e37a48a5ba8230ff3507d345ea07c8897a8c39885" in 3.134s (3.134s including waiting). Image size: 301160875 bytes.' metadata: creationTimestamp: "2026-04-24T22:38:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:50Z" name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m.18a96c0cd685a716 namespace: kserve-ci-e2e-test resourceVersion: "13543" uid: 89ad2efa-9a07-459f-893d-4878e212b4e5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: 4d081b6b-d731-4112-85d1-cd2fe0fe77da kind: Event lastTimestamp: "2026-04-24T22:38:50Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T22:38:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:50Z" name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m.18a96c0cd96ab74c namespace: kserve-ci-e2e-test resourceVersion: "13544" uid: 530592d9-0c78-449e-9b77-e930c1c27083 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:50Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: 4d081b6b-d731-4112-85d1-cd2fe0fe77da kind: Event lastTimestamp: "2026-04-24T22:38:50Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T22:38:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:50Z" name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m.18a96c0cd9dc219f namespace: kserve-ci-e2e-test resourceVersion: "13545" uid: 5438bf9d-b6c1-4adc-bc31-6a089cc10606 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: 4d081b6b-d731-4112-85d1-cd2fe0fe77da kind: Event lastTimestamp: "2026-04-24T22:38:54Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1443" metadata: creationTimestamp: "2026-04-24T22:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:54Z" name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m.18a96c0dcef28d90 namespace: kserve-ci-e2e-test resourceVersion: "13585" uid: ecc1c561-2ba3-4284-8500-8a585c0b10d3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: 4d081b6b-d731-4112-85d1-cd2fe0fe77da kind: Event lastTimestamp: "2026-04-24T22:39:07Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1443" in 12.683s (12.683s including waiting). Image size: 1561011115 bytes.' metadata: creationTimestamp: "2026-04-24T22:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:07Z" name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m.18a96c10c2eb059d namespace: kserve-ci-e2e-test resourceVersion: "13755" uid: ce50e2e7-b5d7-4b72-9115-9a5860abd265 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: 4d081b6b-d731-4112-85d1-cd2fe0fe77da kind: Event lastTimestamp: "2026-04-24T22:39:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:07Z" name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m.18a96c10c64ccc09 namespace: kserve-ci-e2e-test resourceVersion: "13756" uid: 14016174-f8cb-495e-bed3-e7877b3dca80 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: 4d081b6b-d731-4112-85d1-cd2fe0fe77da kind: Event lastTimestamp: "2026-04-24T22:39:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:07Z" name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m.18a96c10c6be331c namespace: kserve-ci-e2e-test resourceVersion: "13757" uid: fe574233-ceca-45f8-a704-673ddfb925d2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: 4d081b6b-d731-4112-85d1-cd2fe0fe77da kind: Event lastTimestamp: "2026-04-24T22:39:07Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T22:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:07Z" name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m.18a96c10c6c92659 namespace: kserve-ci-e2e-test resourceVersion: "13758" uid: 605db2dc-e45b-4389-8392-d85ff41d9395 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: 4d081b6b-d731-4112-85d1-cd2fe0fe77da kind: Event lastTimestamp: "2026-04-24T22:39:09Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.195s (2.195s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T22:39:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:09Z" name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m.18a96c1149ac6e9e namespace: kserve-ci-e2e-test resourceVersion: "13777" uid: 004ce254-06f6-4213-8ef2-11dadce4ebfd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: 4d081b6b-d731-4112-85d1-cd2fe0fe77da kind: Event lastTimestamp: "2026-04-24T22:39:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:39:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:09Z" name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m.18a96c114ce8d5e3 namespace: kserve-ci-e2e-test resourceVersion: "13778" uid: fa37ed29-4a0e-44ff-a5d3-e7c1373c8f06 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: 4d081b6b-d731-4112-85d1-cd2fe0fe77da kind: Event lastTimestamp: "2026-04-24T22:39:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:39:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:09Z" name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m.18a96c114d6c4edf namespace: kserve-ci-e2e-test resourceVersion: "13779" uid: f25c0164-ae0e-4522-ab85-c440ed687dd6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-24T22:39:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: 4d081b6b-d731-4112-85d1-cd2fe0fe77da kind: Event lastTimestamp: "2026-04-24T22:40:06Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:39:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:06Z" name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m.18a96c118c4577ee namespace: kserve-ci-e2e-test resourceVersion: "14271" uid: 90ba2360-07c1-429b-9f35-6c178c0c8b44 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: 4d081b6b-d731-4112-85d1-cd2fe0fe77da kind: Event lastTimestamp: "2026-04-24T22:40:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:57Z" name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m.18a96c2a52ba6273 namespace: kserve-ci-e2e-test resourceVersion: "15030" uid: e86290d4-08aa-4825-9178-1b0111c37f1d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m namespace: kserve-ci-e2e-test resourceVersion: "13373" uid: 4d081b6b-d731-4112-85d1-cd2fe0fe77da kind: Event lastTimestamp: "2026-04-24T22:40:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:57Z" name: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m.18a96c2a52baf41e namespace: kserve-ci-e2e-test resourceVersion: "15034" uid: a1f35d4f-5f94-45e1-86ee-6ef1d9fad0e6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-56dcdd5cdf namespace: kserve-ci-e2e-test resourceVersion: "13359" uid: a8a9c590-7cba-4880-9bb7-f7affbda4cb6 kind: Event lastTimestamp: "2026-04-24T22:38:46Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-56dcdd5cdf-fdd6m' metadata: creationTimestamp: "2026-04-24T22:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:38:46Z" name: isvc-sklearn-graph-1-predictor-56dcdd5cdf.18a96c0bfff019a1 namespace: kserve-ci-e2e-test resourceVersion: "13370" uid: a6d209fb-f797-4838-831a-ba871a41fc98 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13358" uid: fad1319b-b484-435d-b787-8c6885c47b32 kind: Event lastTimestamp: "2026-04-24T22:38:46Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-56dcdd5cdf from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:38:46Z" name: isvc-sklearn-graph-1-predictor.18a96c0bfa74640f namespace: kserve-ci-e2e-test resourceVersion: "13361" uid: 53f47b33-b443-439c-bf2c-85f722760eef reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:39:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13372" uid: 8c886329-b62b-479d-b96b-2799a3dc25f2 kind: Event lastTimestamp: "2026-04-24T22:39:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:39:46Z" name: isvc-sklearn-graph-1-predictor.18a96c0f80602f10 namespace: kserve-ci-e2e-test resourceVersion: "14092" uid: 865adf5c-78c2-4c8c-a963-c0dfd4f82394 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:39:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13372" uid: 8c886329-b62b-479d-b96b-2799a3dc25f2 kind: Event lastTimestamp: "2026-04-24T22:39:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:39:46Z" name: isvc-sklearn-graph-1-predictor.18a96c0f80aaf590 namespace: kserve-ci-e2e-test resourceVersion: "14093" uid: 25ffb39f-8a18-4ea1-86b1-52dfb35bbd8a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:39:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13667" uid: 8c886329-b62b-479d-b96b-2799a3dc25f2 kind: Event lastTimestamp: "2026-04-24T22:40:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:39:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:46Z" name: isvc-sklearn-graph-1-predictor.18a96c167ee0ee27 namespace: kserve-ci-e2e-test resourceVersion: "14842" uid: 08bdab41-d0a0-4885-aece-b2eba7fbc862 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:39:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13667" uid: 8c886329-b62b-479d-b96b-2799a3dc25f2 kind: Event lastTimestamp: "2026-04-24T22:40:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:39:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:46Z" name: isvc-sklearn-graph-1-predictor.18a96c167f4d3aa3 namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: bf24d6eb-5bb6-4e3f-b4cf-e377765ccfae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T22:38:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13340" uid: acf8f7d9-df54-4aa9-8248-727e5252aee0 kind: Event lastTimestamp: "2026-04-24T22:40:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:40:16Z" name: isvc-sklearn-graph-1.18a96c0c00937db9 namespace: kserve-ci-e2e-test resourceVersion: "14399" uid: 4ef2bf50-b0dd-4971-8434-56de57d1a186 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13340" uid: acf8f7d9-df54-4aa9-8248-727e5252aee0 kind: Event lastTimestamp: "2026-04-24T22:38:46Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:38:46Z" name: isvc-sklearn-graph-1.18a96c0c01200229 namespace: kserve-ci-e2e-test resourceVersion: "13378" uid: d59cfb04-6158-4ddb-aaf8-d6d774eb4fe5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13340" uid: acf8f7d9-df54-4aa9-8248-727e5252aee0 kind: Event lastTimestamp: "2026-04-24T22:38:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:38:46Z" name: isvc-sklearn-graph-1.18a96c0c012047c0 namespace: kserve-ci-e2e-test resourceVersion: "13380" uid: 78bd541b-16ad-4916-9543-db21cde759c5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14397" uid: acf8f7d9-df54-4aa9-8248-727e5252aee0 kind: Event lastTimestamp: "2026-04-24T22:40:16Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-24T22:40:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:40:16Z" name: isvc-sklearn-graph-1.18a96c20ef25f19a namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: bf90ac25-d5cc-4064-8063-904fab85dd15 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:38:48.043032Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686 namespace: kserve-ci-e2e-test resourceVersion: "13499" uid: 4ff3301f-15eb-4fca-a9cf-0ef15031bd39 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-7db9dc6879-jr686 to ip-10-0-132-138.ec2.internal metadata: creationTimestamp: "2026-04-24T22:38:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:38:48Z" name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686.18a96c0c4be46ac0 namespace: kserve-ci-e2e-test resourceVersion: "13503" uid: 93350df9-7dd1-46fe-a654-212a09181777 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:48Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686 namespace: kserve-ci-e2e-test resourceVersion: "13507" uid: 4ff3301f-15eb-4fca-a9cf-0ef15031bd39 kind: Event lastTimestamp: "2026-04-24T22:38:48Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:38:48Z" name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686.18a96c0c68f5b704 namespace: kserve-ci-e2e-test resourceVersion: "13518" uid: f3e78d6b-5fce-4ad1-b6d4-8c07e1fb8185 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686 namespace: kserve-ci-e2e-test resourceVersion: "13501" uid: 4ff3301f-15eb-4fca-a9cf-0ef15031bd39 kind: Event lastTimestamp: "2026-04-24T22:38:48Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:4ae4df75945dd799b0c11b5e37a48a5ba8230ff3507d345ea07c8897a8c39885" metadata: creationTimestamp: "2026-04-24T22:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:48Z" name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686.18a96c0c6ac0c213 namespace: kserve-ci-e2e-test resourceVersion: "13522" uid: 34b1fb0b-fae3-4028-89eb-a1f5e62c3833 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686 namespace: kserve-ci-e2e-test resourceVersion: "13501" uid: 4ff3301f-15eb-4fca-a9cf-0ef15031bd39 kind: Event lastTimestamp: "2026-04-24T22:39:00Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:4ae4df75945dd799b0c11b5e37a48a5ba8230ff3507d345ea07c8897a8c39885" in 12.108s (12.108s including waiting). Image size: 301160875 bytes.' metadata: creationTimestamp: "2026-04-24T22:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:00Z" name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686.18a96c0f3c78fe3f namespace: kserve-ci-e2e-test resourceVersion: "13641" uid: 8dbef0a6-235e-4d2e-9f47-fe37ff616143 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686 namespace: kserve-ci-e2e-test resourceVersion: "13501" uid: 4ff3301f-15eb-4fca-a9cf-0ef15031bd39 kind: Event lastTimestamp: "2026-04-24T22:39:00Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T22:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:00Z" name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686.18a96c0f3fb08e88 namespace: kserve-ci-e2e-test resourceVersion: "13642" uid: 23bce8d6-82eb-4bed-a3d7-a096e117b092 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686 namespace: kserve-ci-e2e-test resourceVersion: "13501" uid: 4ff3301f-15eb-4fca-a9cf-0ef15031bd39 kind: Event lastTimestamp: "2026-04-24T22:39:00Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T22:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:00Z" name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686.18a96c0f4027112e namespace: kserve-ci-e2e-test resourceVersion: "13643" uid: 05f324b7-0207-44b0-b346-5ebb944517df reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686 namespace: kserve-ci-e2e-test resourceVersion: "13501" uid: 4ff3301f-15eb-4fca-a9cf-0ef15031bd39 kind: Event lastTimestamp: "2026-04-24T22:39:05Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1443" metadata: creationTimestamp: "2026-04-24T22:39:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:05Z" name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686.18a96c1054731d61 namespace: kserve-ci-e2e-test resourceVersion: "13730" uid: 7f67486d-45ab-43aa-b7d4-046ddf921238 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686 namespace: kserve-ci-e2e-test resourceVersion: "13501" uid: 4ff3301f-15eb-4fca-a9cf-0ef15031bd39 kind: Event lastTimestamp: "2026-04-24T22:39:10Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1443" in 5.547s (5.547s including waiting). Image size: 1561011115 bytes.' metadata: creationTimestamp: "2026-04-24T22:39:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:10Z" name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686.18a96c119f214780 namespace: kserve-ci-e2e-test resourceVersion: "13789" uid: 1b3cab2f-47e7-41de-bfb2-9e1db24746ff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686 namespace: kserve-ci-e2e-test resourceVersion: "13501" uid: 4ff3301f-15eb-4fca-a9cf-0ef15031bd39 kind: Event lastTimestamp: "2026-04-24T22:39:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:39:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:10Z" name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686.18a96c11a28aee92 namespace: kserve-ci-e2e-test resourceVersion: "13790" uid: c36337b7-927f-4e56-a90d-27b06b72e824 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686 namespace: kserve-ci-e2e-test resourceVersion: "13501" uid: 4ff3301f-15eb-4fca-a9cf-0ef15031bd39 kind: Event lastTimestamp: "2026-04-24T22:39:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:39:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:10Z" name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686.18a96c11a3014f20 namespace: kserve-ci-e2e-test resourceVersion: "13791" uid: 75df494a-7834-4477-a3ab-0aefdc2ad275 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686 namespace: kserve-ci-e2e-test resourceVersion: "13501" uid: 4ff3301f-15eb-4fca-a9cf-0ef15031bd39 kind: Event lastTimestamp: "2026-04-24T22:39:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:39:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:10Z" name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686.18a96c11a31a98b2 namespace: kserve-ci-e2e-test resourceVersion: "13792" uid: d2f50e8e-349d-4603-9edf-5b2466b5ee04 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686 namespace: kserve-ci-e2e-test resourceVersion: "13501" uid: 4ff3301f-15eb-4fca-a9cf-0ef15031bd39 kind: Event lastTimestamp: "2026-04-24T22:39:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:11Z" name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686.18a96c11a622f54a namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: 301a38df-f0b4-47ca-ab00-584d35b10070 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686 namespace: kserve-ci-e2e-test resourceVersion: "13501" uid: 4ff3301f-15eb-4fca-a9cf-0ef15031bd39 kind: Event lastTimestamp: "2026-04-24T22:39:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:11Z" name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686.18a96c11a696a20c namespace: kserve-ci-e2e-test resourceVersion: "13795" uid: daaeb153-b9ba-4342-8cba-a7cd42b30495 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T22:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686 namespace: kserve-ci-e2e-test resourceVersion: "13501" uid: 4ff3301f-15eb-4fca-a9cf-0ef15031bd39 kind: Event lastTimestamp: "2026-04-24T22:40:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:58Z" name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686.18a96c11f7948897 namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: 468a3e52-1012-4797-95f9-e8a7f9a3f65e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686 namespace: kserve-ci-e2e-test resourceVersion: "13501" uid: 4ff3301f-15eb-4fca-a9cf-0ef15031bd39 kind: Event lastTimestamp: "2026-04-24T22:40:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:40:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:56Z" name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686.18a96c2a4ef9de07 namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: e04af4cb-b5d5-4315-a25c-b08ba277e4ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686 namespace: kserve-ci-e2e-test resourceVersion: "13501" uid: 4ff3301f-15eb-4fca-a9cf-0ef15031bd39 kind: Event lastTimestamp: "2026-04-24T22:40:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:40:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:56Z" name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686.18a96c2a4efa9118 namespace: kserve-ci-e2e-test resourceVersion: "15001" uid: c61f820b-f252-4d65-a0de-d855bd12281e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686 namespace: kserve-ci-e2e-test resourceVersion: "13501" uid: 4ff3301f-15eb-4fca-a9cf-0ef15031bd39 kind: Event lastTimestamp: "2026-04-24T22:40:58Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:58Z" name: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686.18a96c2aa5da7022 namespace: kserve-ci-e2e-test resourceVersion: "15097" uid: 0c3dfa77-e470-48d9-b9dd-bf3a427879ea reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-138.ec2.internal source: component: kubelet host: ip-10-0-132-138.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-7db9dc6879 namespace: kserve-ci-e2e-test resourceVersion: "13480" uid: c049d602-0705-4ba3-bbd6-53404a598109 kind: Event lastTimestamp: "2026-04-24T22:38:48Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-7db9dc6879-jr686' metadata: creationTimestamp: "2026-04-24T22:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:38:48Z" name: isvc-sklearn-graph-2-predictor-7db9dc6879.18a96c0c4b7f487e namespace: kserve-ci-e2e-test resourceVersion: "13500" uid: 42c9cd83-da1b-4221-8f5b-aa431e27145a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13478" uid: da2e2822-5884-4c1f-8078-acd2c5d673ea kind: Event lastTimestamp: "2026-04-24T22:38:47Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-7db9dc6879 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:38:47Z" name: isvc-sklearn-graph-2-predictor.18a96c0c322213a2 namespace: kserve-ci-e2e-test resourceVersion: "13482" uid: 68389905-2892-48ff-8e50-2a80a9f7494f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T22:39:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13479" uid: f9455245-3df3-4853-bafe-07026288e28b kind: Event lastTimestamp: "2026-04-24T22:39:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:39:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:39:47Z" name: isvc-sklearn-graph-2-predictor.18a96c0fb0a24736 namespace: kserve-ci-e2e-test resourceVersion: "14111" uid: df8d08a9-49e2-4d54-87b6-3e612395c406 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T22:39:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13479" uid: f9455245-3df3-4853-bafe-07026288e28b kind: Event lastTimestamp: "2026-04-24T22:39:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:39:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:39:47Z" name: isvc-sklearn-graph-2-predictor.18a96c0fb11499ab namespace: kserve-ci-e2e-test resourceVersion: "14112" uid: d7441e2a-1413-4321-8615-834b3ccba7e2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T22:40:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13692" uid: f9455245-3df3-4853-bafe-07026288e28b kind: Event lastTimestamp: "2026-04-24T22:40:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:47Z" name: isvc-sklearn-graph-2-predictor.18a96c1dac12e939 namespace: kserve-ci-e2e-test resourceVersion: "14858" uid: 3c9f2e43-3477-4765-a286-21a943c8e849 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T22:40:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13692" uid: f9455245-3df3-4853-bafe-07026288e28b kind: Event lastTimestamp: "2026-04-24T22:40:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:47Z" name: isvc-sklearn-graph-2-predictor.18a96c1dac5bd897 namespace: kserve-ci-e2e-test resourceVersion: "14859" uid: 8b34a85e-4503-40e3-bcb8-bd4db7cfb573 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T22:38:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13465" uid: 87057121-5142-489f-bb23-9f3dfe38c331 kind: Event lastTimestamp: "2026-04-24T22:40:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:40:19Z" name: isvc-sklearn-graph-2.18a96c0c31f5e30a namespace: kserve-ci-e2e-test resourceVersion: "14575" uid: 1e653694-89c1-42f6-8d31-d69307b484bf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14565" uid: 87057121-5142-489f-bb23-9f3dfe38c331 kind: Event lastTimestamp: "2026-04-24T22:40:18Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-24T22:40:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:40:19Z" name: isvc-sklearn-graph-2.18a96c21781961d7 namespace: kserve-ci-e2e-test resourceVersion: "14568" uid: f7c93a0c-a05c-469d-9b7d-660cfdc89e77 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:38:47.148754Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2 namespace: kserve-ci-e2e-test resourceVersion: "13442" uid: 79c9f152-cc03-467c-b33e-c1c8523e218d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-p5mg2 to ip-10-0-142-202.ec2.internal metadata: creationTimestamp: "2026-04-24T22:38:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:38:47Z" name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2.18a96c0c1696d34a namespace: kserve-ci-e2e-test resourceVersion: "13447" uid: e3bd6b8d-0582-4827-945c-bc3a2d9724ec reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:47Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2 namespace: kserve-ci-e2e-test resourceVersion: "13445" uid: 79c9f152-cc03-467c-b33e-c1c8523e218d kind: Event lastTimestamp: "2026-04-24T22:38:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T22:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:47Z" name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2.18a96c0c22020fae namespace: kserve-ci-e2e-test resourceVersion: "13468" uid: 8e7c3000-3011-49de-aec5-089b450d9238 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:48Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2 namespace: kserve-ci-e2e-test resourceVersion: "13451" uid: 79c9f152-cc03-467c-b33e-c1c8523e218d kind: Event lastTimestamp: "2026-04-24T22:38:48Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:38:48Z" name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2.18a96c0c5336ace0 namespace: kserve-ci-e2e-test resourceVersion: "13509" uid: 5ec82a8d-b111-46e0-929b-69ec62554988 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:48Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2 namespace: kserve-ci-e2e-test resourceVersion: "13445" uid: 79c9f152-cc03-467c-b33e-c1c8523e218d kind: Event lastTimestamp: "2026-04-24T22:38:48Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:4ae4df75945dd799b0c11b5e37a48a5ba8230ff3507d345ea07c8897a8c39885" metadata: creationTimestamp: "2026-04-24T22:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:48Z" name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2.18a96c0c547a1706 namespace: kserve-ci-e2e-test resourceVersion: "13512" uid: 21b5fb7c-446b-4420-876e-1ea6566679b5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2 namespace: kserve-ci-e2e-test resourceVersion: "13445" uid: 79c9f152-cc03-467c-b33e-c1c8523e218d kind: Event lastTimestamp: "2026-04-24T22:39:01Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:4ae4df75945dd799b0c11b5e37a48a5ba8230ff3507d345ea07c8897a8c39885" in 13.438s (13.438s including waiting). Image size: 301160875 bytes.' metadata: creationTimestamp: "2026-04-24T22:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:01Z" name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2.18a96c0f7580e612 namespace: kserve-ci-e2e-test resourceVersion: "13659" uid: 96dfb029-28b1-4a06-8664-631f5f457078 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2 namespace: kserve-ci-e2e-test resourceVersion: "13445" uid: 79c9f152-cc03-467c-b33e-c1c8523e218d kind: Event lastTimestamp: "2026-04-24T22:39:01Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-24T22:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:01Z" name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2.18a96c0f787d8cdf namespace: kserve-ci-e2e-test resourceVersion: "13660" uid: dea51eb7-8ea8-4f6e-9246-2d5fe0c93864 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2 namespace: kserve-ci-e2e-test resourceVersion: "13445" uid: 79c9f152-cc03-467c-b33e-c1c8523e218d kind: Event lastTimestamp: "2026-04-24T22:39:01Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-24T22:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:01Z" name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2.18a96c0f78f5fbea namespace: kserve-ci-e2e-test resourceVersion: "13661" uid: c1161fbf-460b-4b9a-9dfe-1feb91bf61bb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2 namespace: kserve-ci-e2e-test resourceVersion: "13445" uid: 79c9f152-cc03-467c-b33e-c1c8523e218d kind: Event lastTimestamp: "2026-04-24T22:39:06Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-24T22:39:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:06Z" name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2.18a96c1080e60555 namespace: kserve-ci-e2e-test resourceVersion: "13739" uid: d464a304-c13f-4045-8bfc-a7cbaa02518c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2 namespace: kserve-ci-e2e-test resourceVersion: "13445" uid: 79c9f152-cc03-467c-b33e-c1c8523e218d kind: Event lastTimestamp: "2026-04-24T22:39:23Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 17.491s (17.491s including waiting). Image size: 1306417402 bytes.' metadata: creationTimestamp: "2026-04-24T22:39:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:23Z" name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2.18a96c149379ab0c namespace: kserve-ci-e2e-test resourceVersion: "13903" uid: ca7beaa4-2759-4c73-8d9f-92a4866b5483 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2 namespace: kserve-ci-e2e-test resourceVersion: "13445" uid: 79c9f152-cc03-467c-b33e-c1c8523e218d kind: Event lastTimestamp: "2026-04-24T22:39:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:39:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:23Z" name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2.18a96c149931cadf namespace: kserve-ci-e2e-test resourceVersion: "13904" uid: 7ad7ae5e-0be7-40c4-a71a-461e8b265629 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2 namespace: kserve-ci-e2e-test resourceVersion: "13445" uid: 79c9f152-cc03-467c-b33e-c1c8523e218d kind: Event lastTimestamp: "2026-04-24T22:39:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:39:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:23Z" name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2.18a96c149a1bff86 namespace: kserve-ci-e2e-test resourceVersion: "13905" uid: 8093ab31-2169-4ece-b35e-96bbddcae149 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2 namespace: kserve-ci-e2e-test resourceVersion: "13445" uid: 79c9f152-cc03-467c-b33e-c1c8523e218d kind: Event lastTimestamp: "2026-04-24T22:39:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:39:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:23Z" name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2.18a96c149a28cb1d namespace: kserve-ci-e2e-test resourceVersion: "13906" uid: 12dee2b7-a3a9-492c-9fb4-c7bc8667c56d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2 namespace: kserve-ci-e2e-test resourceVersion: "13445" uid: 79c9f152-cc03-467c-b33e-c1c8523e218d kind: Event lastTimestamp: "2026-04-24T22:39:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:39:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:23Z" name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2.18a96c149de09c46 namespace: kserve-ci-e2e-test resourceVersion: "13907" uid: e4c8d822-3a6b-4e39-89c9-c0abc06011d9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2 namespace: kserve-ci-e2e-test resourceVersion: "13445" uid: 79c9f152-cc03-467c-b33e-c1c8523e218d kind: Event lastTimestamp: "2026-04-24T22:39:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:39:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:23Z" name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2.18a96c149f4d400c namespace: kserve-ci-e2e-test resourceVersion: "13908" uid: 63c7e281-23b1-44e1-8335-b958cb454001 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-24T22:39:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2 namespace: kserve-ci-e2e-test resourceVersion: "13445" uid: 79c9f152-cc03-467c-b33e-c1c8523e218d kind: Event lastTimestamp: "2026-04-24T22:41:00Z" message: 'Readiness probe failed: dial tcp 10.132.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:41:00Z" name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2.18a96c14b50ac383 namespace: kserve-ci-e2e-test resourceVersion: "15119" uid: 4f33b90f-e174-40e5-b708-1f5597fd3b42 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2 namespace: kserve-ci-e2e-test resourceVersion: "13445" uid: 79c9f152-cc03-467c-b33e-c1c8523e218d kind: Event lastTimestamp: "2026-04-24T22:40:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:40:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:56Z" name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2.18a96c2a4baa4fb5 namespace: kserve-ci-e2e-test resourceVersion: "14971" uid: 9947b301-5ae0-4fcd-9825-77c92bff30cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2 namespace: kserve-ci-e2e-test resourceVersion: "13445" uid: 79c9f152-cc03-467c-b33e-c1c8523e218d kind: Event lastTimestamp: "2026-04-24T22:40:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:40:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:56Z" name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2.18a96c2a4bab0c1e namespace: kserve-ci-e2e-test resourceVersion: "14973" uid: 1e108d84-4454-4404-8c72-05b86d7d9111 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2 namespace: kserve-ci-e2e-test resourceVersion: "13445" uid: 79c9f152-cc03-467c-b33e-c1c8523e218d kind: Event lastTimestamp: "2026-04-24T22:41:00Z" message: 'Readiness probe failed: Get "https://10.132.0.18:8643/healthz": dial tcp 10.132.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:41:00Z" name: isvc-xgboost-graph-predictor-669d8d6456-p5mg2.18a96c2b0f3d88b5 namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 58bd19e4-9ce8-4d96-9658-29913aba0507 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "13417" uid: 15d343f2-021c-4c25-9afb-5c048075aaca kind: Event lastTimestamp: "2026-04-24T22:38:47Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-p5mg2' metadata: creationTimestamp: "2026-04-24T22:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:38:47Z" name: isvc-xgboost-graph-predictor-669d8d6456.18a96c0c1605c0db namespace: kserve-ci-e2e-test resourceVersion: "13443" uid: b3871cfc-dc7d-4823-905d-8b7bcbd1b644 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13415" uid: 4e0b2748-7073-45c4-9750-43aebe1e3681 kind: Event lastTimestamp: "2026-04-24T22:38:46Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:38:46Z" name: isvc-xgboost-graph-predictor.18a96c0c0ba76427 namespace: kserve-ci-e2e-test resourceVersion: "13419" uid: ceb29d0a-d4f9-4b6f-a75f-5382673f98e3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T22:39:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13416" uid: 1a3df651-a88c-406d-8f18-06163f91ca16 kind: Event lastTimestamp: "2026-04-24T22:40:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:02Z" name: isvc-xgboost-graph-predictor.18a96c0f8a22db34 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: a82da86e-c5da-4bd6-b952-ee9f7446cf91 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T22:39:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13416" uid: 1a3df651-a88c-406d-8f18-06163f91ca16 kind: Event lastTimestamp: "2026-04-24T22:40:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:02Z" name: isvc-xgboost-graph-predictor.18a96c0f8a8925fa namespace: kserve-ci-e2e-test resourceVersion: "14235" uid: b115852f-6007-4ce8-8ed1-f0b10ffd6592 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T22:39:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: 1a3df651-a88c-406d-8f18-06163f91ca16 kind: Event lastTimestamp: "2026-04-24T22:40:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:47Z" name: isvc-xgboost-graph-predictor.18a96c1a071b61b2 namespace: kserve-ci-e2e-test resourceVersion: "14848" uid: fbe3a8d9-8e46-457a-842b-223b16271520 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T22:39:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13676" uid: 1a3df651-a88c-406d-8f18-06163f91ca16 kind: Event lastTimestamp: "2026-04-24T22:40:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:47Z" name: isvc-xgboost-graph-predictor.18a96c1a07bf075f namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: 77fe6888-f0b1-484a-869d-c34f13ef9df1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-24T22:38:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13400" uid: 28e63278-5441-49dc-a560-b858e9ba273a kind: Event lastTimestamp: "2026-04-24T22:40:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:40:30Z" name: isvc-xgboost-graph.18a96c0c0b801942 namespace: kserve-ci-e2e-test resourceVersion: "14672" uid: beb2f437-c909-476a-98b6-0e355afcc533 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14670" uid: 28e63278-5441-49dc-a560-b858e9ba273a kind: Event lastTimestamp: "2026-04-24T22:40:30Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-24T22:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:40:30Z" name: isvc-xgboost-graph.18a96c24185b9950 namespace: kserve-ci-e2e-test resourceVersion: "14671" uid: 1d3a6d4a-ceb0-4af2-9201-ba6949b9eb6b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:40:46.715915Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-85d456f99f-98r87 namespace: kserve-ci-e2e-test resourceVersion: "14823" uid: 33b7f016-da48-4a91-9397-9afab6a1c89c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-85d456f99f-98r87 to ip-10-0-133-161.ec2.internal metadata: creationTimestamp: "2026-04-24T22:40:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:40:46Z" name: model-chainer-85d456f99f-98r87.18a96c27ed58e0c2 namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: 655ec89b-b7ab-4cce-b83f-863ebaae0236 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:46Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-85d456f99f-98r87 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 33b7f016-da48-4a91-9397-9afab6a1c89c kind: Event lastTimestamp: "2026-04-24T22:40:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T22:40:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:46Z" name: model-chainer-85d456f99f-98r87.18a96c27f66f8a65 namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: 43041656-d0a1-4b4d-be21-6e968df9e246 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:47Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-85d456f99f-98r87 namespace: kserve-ci-e2e-test resourceVersion: "14832" uid: 33b7f016-da48-4a91-9397-9afab6a1c89c kind: Event lastTimestamp: "2026-04-24T22:40:47Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:40:47Z" name: model-chainer-85d456f99f-98r87.18a96c2829b8a9e7 namespace: kserve-ci-e2e-test resourceVersion: "14860" uid: ec4c0697-95a0-421b-a450-098c45dd6a66 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-85d456f99f-98r87 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 33b7f016-da48-4a91-9397-9afab6a1c89c kind: Event lastTimestamp: "2026-04-24T22:40:47Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f3ef58ae57fd8b984890d9aeb7935958f0ea100fbddeb0833d76b7e2007120ff" already present on machine metadata: creationTimestamp: "2026-04-24T22:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:47Z" name: model-chainer-85d456f99f-98r87.18a96c282affe04b namespace: kserve-ci-e2e-test resourceVersion: "14863" uid: c5f23f7c-8295-47f6-89ff-95a2f590f70f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-85d456f99f-98r87 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 33b7f016-da48-4a91-9397-9afab6a1c89c kind: Event lastTimestamp: "2026-04-24T22:40:47Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-24T22:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:47Z" name: model-chainer-85d456f99f-98r87.18a96c282dd5ab26 namespace: kserve-ci-e2e-test resourceVersion: "14864" uid: 269e001c-18a7-48cc-8e81-cc5b51e930c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-85d456f99f-98r87 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 33b7f016-da48-4a91-9397-9afab6a1c89c kind: Event lastTimestamp: "2026-04-24T22:40:47Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-24T22:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:47Z" name: model-chainer-85d456f99f-98r87.18a96c282e4564fb namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: e34421dc-0f11-4d3f-a616-5611f75ca954 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-85d456f99f-98r87 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 33b7f016-da48-4a91-9397-9afab6a1c89c kind: Event lastTimestamp: "2026-04-24T22:40:56Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-24T22:40:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:56Z" name: model-chainer-85d456f99f-98r87.18a96c2a47daa063 namespace: kserve-ci-e2e-test resourceVersion: "14952" uid: 20f02cdf-5f10-405f-b8a7-62a6fc6feae1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-85d456f99f-98r87 namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 33b7f016-da48-4a91-9397-9afab6a1c89c kind: Event lastTimestamp: "2026-04-24T22:41:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T22:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:41:23Z" name: model-chainer-85d456f99f-98r87.18a96c2ac3ec7a3c namespace: kserve-ci-e2e-test resourceVersion: "15379" uid: 77639bdd-a11e-44e6-a948-ff0820d68fc4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-85d456f99f namespace: kserve-ci-e2e-test resourceVersion: "14820" uid: 590dd307-ea77-4a29-8f8f-0c3ba29d0479 kind: Event lastTimestamp: "2026-04-24T22:40:46Z" message: 'Created pod: model-chainer-85d456f99f-98r87' metadata: creationTimestamp: "2026-04-24T22:40:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:46Z" name: model-chainer-85d456f99f.18a96c27ed05de0f namespace: kserve-ci-e2e-test resourceVersion: "14825" uid: 7faa4617-8c99-4cea-93f5-a409694cda23 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 619cac96-bf1d-45bc-9952-d4a6b6f0ac3b kind: Event lastTimestamp: "2026-04-24T22:40:46Z" message: Scaled up replica set model-chainer-85d456f99f from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:40:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:46Z" name: model-chainer.18a96c27ec24daa8 namespace: kserve-ci-e2e-test resourceVersion: "14821" uid: 2a48a4cd-4039-430f-bc12-6d1d6bffb315 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: 2feffcd5-b048-4b22-bbfc-2a53a1221cd3 kind: Event lastTimestamp: "2026-04-24T22:40:46Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-24T22:40:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:40:46Z" name: model-chainer.18a96c27f1781610 namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: a2b8cfb5-46cd-403e-a00d-6ab9ff16157e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:52:22.237778Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2e6fe-dcb8bc6f8-mvk48 namespace: kserve-ci-e2e-test resourceVersion: "20918" uid: c8c55da2-39df-4339-8f89-142de9271828 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-2e6fe-dcb8bc6f8-mvk48 to ip-10-0-133-161.ec2.internal metadata: creationTimestamp: "2026-04-24T22:52:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:52:22Z" name: sequence-graph-2e6fe-dcb8bc6f8-mvk48.18a96cc9ddae4901 namespace: kserve-ci-e2e-test resourceVersion: "20924" uid: 51a30d3e-a6c6-40e1-b2ab-8648282e5e93 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:52:22Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2e6fe-dcb8bc6f8-mvk48 namespace: kserve-ci-e2e-test resourceVersion: "20920" uid: c8c55da2-39df-4339-8f89-142de9271828 kind: Event lastTimestamp: "2026-04-24T22:52:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-2e6fe-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T22:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:52:22Z" name: sequence-graph-2e6fe-dcb8bc6f8-mvk48.18a96cc9ea744d4d namespace: kserve-ci-e2e-test resourceVersion: "20937" uid: 40095a07-d03e-4915-b268-6a8f1c575d18 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:52:23Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2e6fe-dcb8bc6f8-mvk48 namespace: kserve-ci-e2e-test resourceVersion: "20928" uid: c8c55da2-39df-4339-8f89-142de9271828 kind: Event lastTimestamp: "2026-04-24T22:52:23Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:52:23Z" name: sequence-graph-2e6fe-dcb8bc6f8-mvk48.18a96cca1a876446 namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: b20a3026-6001-49f2-a4d2-a63435bc26d9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:52:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2e6fe} kind: Pod name: sequence-graph-2e6fe-dcb8bc6f8-mvk48 namespace: kserve-ci-e2e-test resourceVersion: "20920" uid: c8c55da2-39df-4339-8f89-142de9271828 kind: Event lastTimestamp: "2026-04-24T22:52:23Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f3ef58ae57fd8b984890d9aeb7935958f0ea100fbddeb0833d76b7e2007120ff" already present on machine metadata: creationTimestamp: "2026-04-24T22:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:52:23Z" name: sequence-graph-2e6fe-dcb8bc6f8-mvk48.18a96cca1bbd3900 namespace: kserve-ci-e2e-test resourceVersion: "20947" uid: 4ac2590e-a9ed-4598-96b1-c1da06e7d515 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:52:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2e6fe} kind: Pod name: sequence-graph-2e6fe-dcb8bc6f8-mvk48 namespace: kserve-ci-e2e-test resourceVersion: "20920" uid: c8c55da2-39df-4339-8f89-142de9271828 kind: Event lastTimestamp: "2026-04-24T22:52:23Z" message: 'Created container: sequence-graph-2e6fe' metadata: creationTimestamp: "2026-04-24T22:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:52:23Z" name: sequence-graph-2e6fe-dcb8bc6f8-mvk48.18a96cca1e93120c namespace: kserve-ci-e2e-test resourceVersion: "20948" uid: 64e21cd6-5b03-4114-ac70-13b8dd13c655 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:52:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2e6fe} kind: Pod name: sequence-graph-2e6fe-dcb8bc6f8-mvk48 namespace: kserve-ci-e2e-test resourceVersion: "20920" uid: c8c55da2-39df-4339-8f89-142de9271828 kind: Event lastTimestamp: "2026-04-24T22:52:23Z" message: Started container sequence-graph-2e6fe metadata: creationTimestamp: "2026-04-24T22:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:52:23Z" name: sequence-graph-2e6fe-dcb8bc6f8-mvk48.18a96cca1f0163c0 namespace: kserve-ci-e2e-test resourceVersion: "20949" uid: d2f9edbc-09a9-423e-8f41-c94607e3704b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2e6fe} kind: Pod name: sequence-graph-2e6fe-dcb8bc6f8-mvk48 namespace: kserve-ci-e2e-test resourceVersion: "20920" uid: c8c55da2-39df-4339-8f89-142de9271828 kind: Event lastTimestamp: "2026-04-24T23:00:36Z" message: Stopping container sequence-graph-2e6fe metadata: creationTimestamp: "2026-04-24T23:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:36Z" name: sequence-graph-2e6fe-dcb8bc6f8-mvk48.18a96d3d0eb2ea25 namespace: kserve-ci-e2e-test resourceVersion: "24526" uid: 1ec31f45-9c09-4f66-bba8-f537f8098dd3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T23:00:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2e6fe} kind: Pod name: sequence-graph-2e6fe-dcb8bc6f8-mvk48 namespace: kserve-ci-e2e-test resourceVersion: "20920" uid: c8c55da2-39df-4339-8f89-142de9271828 kind: Event lastTimestamp: "2026-04-24T23:01:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T23:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:04Z" name: sequence-graph-2e6fe-dcb8bc6f8-mvk48.18a96d3dc05fd1b6 namespace: kserve-ci-e2e-test resourceVersion: "24932" uid: 90bebb54-717a-4d21-b481-c34ebf29fc57 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:52:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-2e6fe-dcb8bc6f8 namespace: kserve-ci-e2e-test resourceVersion: "20914" uid: 28e7a33c-0947-4628-b7c9-2e7664e4a2ff kind: Event lastTimestamp: "2026-04-24T22:52:22Z" message: 'Created pod: sequence-graph-2e6fe-dcb8bc6f8-mvk48' metadata: creationTimestamp: "2026-04-24T22:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:52:22Z" name: sequence-graph-2e6fe-dcb8bc6f8.18a96cc9dd487127 namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: a90ad500-c98d-4216-94d4-62e6b20c63e9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:52:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-2e6fe namespace: kserve-ci-e2e-test resourceVersion: "20912" uid: 210be3fc-c717-4b51-b522-6ccacca97559 kind: Event lastTimestamp: "2026-04-24T22:52:22Z" message: Scaled up replica set sequence-graph-2e6fe-dcb8bc6f8 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:52:22Z" name: sequence-graph-2e6fe.18a96cc9dc8c2528 namespace: kserve-ci-e2e-test resourceVersion: "20916" uid: 39a8aab5-6fe7-4056-b4f4-5dab18acd8f3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:52:22Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-2e6fe namespace: kserve-ci-e2e-test resourceVersion: "20930" uid: 71e534f2-6273-476e-bc9e-b905e800f2cd kind: Event lastTimestamp: "2026-04-24T22:52:22Z" message: InferenceGraph [sequence-graph-2e6fe] is Ready metadata: creationTimestamp: "2026-04-24T22:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:52:22Z" name: sequence-graph-2e6fe.18a96cc9e05ee922 namespace: kserve-ci-e2e-test resourceVersion: "20932" uid: 676999ec-c242-48aa-afcd-1e982dbd0c98 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:52:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-2e6fe namespace: kserve-ci-e2e-test resourceVersion: "20915" uid: 149379f3-2353-4fee-b1e7-7a1765b8f519 kind: Event lastTimestamp: "2026-04-24T22:53:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:52:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:53:07Z" name: sequence-graph-2e6fe.18a96ccd5b368249 namespace: kserve-ci-e2e-test resourceVersion: "21269" uid: 19bc9bc5-adbb-47d3-820f-4e0a8bcebabe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:52:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-2e6fe namespace: kserve-ci-e2e-test resourceVersion: "20915" uid: 149379f3-2353-4fee-b1e7-7a1765b8f519 kind: Event lastTimestamp: "2026-04-24T22:53:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:52:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:53:07Z" name: sequence-graph-2e6fe.18a96ccd5b7c4893 namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: 04b1c73d-9d0f-41df-bc31-8b727e130a29 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:51:11.945993Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-32c7c-6ff7f8b44c-rwdm4 namespace: kserve-ci-e2e-test resourceVersion: "20099" uid: 5159dba1-de60-46b3-92f2-5850e41fb850 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-32c7c-6ff7f8b44c-rwdm4 to ip-10-0-133-161.ec2.internal metadata: creationTimestamp: "2026-04-24T22:51:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:51:11Z" name: sequence-graph-32c7c-6ff7f8b44c-rwdm4.18a96cb97ff6c1a4 namespace: kserve-ci-e2e-test resourceVersion: "20104" uid: aee158ce-b3e3-45cf-8f8d-ccfaf22cf24a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:51:12Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-32c7c-6ff7f8b44c-rwdm4 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: 5159dba1-de60-46b3-92f2-5850e41fb850 kind: Event lastTimestamp: "2026-04-24T22:51:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-32c7c-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T22:51:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:12Z" name: sequence-graph-32c7c-6ff7f8b44c-rwdm4.18a96cb98ce8c69c namespace: kserve-ci-e2e-test resourceVersion: "20131" uid: b01dcf5d-b0f3-4b9b-b292-6207a0f79baa reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:13Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-32c7c-6ff7f8b44c-rwdm4 namespace: kserve-ci-e2e-test resourceVersion: "20108" uid: 5159dba1-de60-46b3-92f2-5850e41fb850 kind: Event lastTimestamp: "2026-04-24T22:51:13Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:51:13Z" name: sequence-graph-32c7c-6ff7f8b44c-rwdm4.18a96cb9f2359275 namespace: kserve-ci-e2e-test resourceVersion: "20145" uid: 8ec31953-1b5a-4b51-9895-56160faf5554 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-32c7c} kind: Pod name: sequence-graph-32c7c-6ff7f8b44c-rwdm4 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: 5159dba1-de60-46b3-92f2-5850e41fb850 kind: Event lastTimestamp: "2026-04-24T22:51:13Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f3ef58ae57fd8b984890d9aeb7935958f0ea100fbddeb0833d76b7e2007120ff" already present on machine metadata: creationTimestamp: "2026-04-24T22:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:13Z" name: sequence-graph-32c7c-6ff7f8b44c-rwdm4.18a96cb9f4750856 namespace: kserve-ci-e2e-test resourceVersion: "20147" uid: 75480b0f-cb93-48cf-84d3-d4f2a824e863 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-32c7c} kind: Pod name: sequence-graph-32c7c-6ff7f8b44c-rwdm4 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: 5159dba1-de60-46b3-92f2-5850e41fb850 kind: Event lastTimestamp: "2026-04-24T22:51:13Z" message: 'Created container: sequence-graph-32c7c' metadata: creationTimestamp: "2026-04-24T22:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:13Z" name: sequence-graph-32c7c-6ff7f8b44c-rwdm4.18a96cb9f77c52f4 namespace: kserve-ci-e2e-test resourceVersion: "20148" uid: 26b5ceaf-3705-4e74-a969-e4ca346681f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-32c7c} kind: Pod name: sequence-graph-32c7c-6ff7f8b44c-rwdm4 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: 5159dba1-de60-46b3-92f2-5850e41fb850 kind: Event lastTimestamp: "2026-04-24T22:51:13Z" message: Started container sequence-graph-32c7c metadata: creationTimestamp: "2026-04-24T22:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:13Z" name: sequence-graph-32c7c-6ff7f8b44c-rwdm4.18a96cb9f7ea8ab9 namespace: kserve-ci-e2e-test resourceVersion: "20149" uid: 58d8d660-1382-43b6-abf0-e47bfcdee14c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-32c7c} kind: Pod name: sequence-graph-32c7c-6ff7f8b44c-rwdm4 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: 5159dba1-de60-46b3-92f2-5850e41fb850 kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: Stopping container sequence-graph-32c7c metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:22Z" name: sequence-graph-32c7c-6ff7f8b44c-rwdm4.18a96cbbd90ae897 namespace: kserve-ci-e2e-test resourceVersion: "20226" uid: 6d68cdf8-f80b-4745-8f61-c00ac6864676 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:51:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-32c7c} kind: Pod name: sequence-graph-32c7c-6ff7f8b44c-rwdm4 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: 5159dba1-de60-46b3-92f2-5850e41fb850 kind: Event lastTimestamp: "2026-04-24T22:51:50Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T22:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:50Z" name: sequence-graph-32c7c-6ff7f8b44c-rwdm4.18a96cbcb740fec4 namespace: kserve-ci-e2e-test resourceVersion: "20642" uid: 02dd2a99-2936-467b-8bec-50a8874baa19 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-32c7c-6ff7f8b44c namespace: kserve-ci-e2e-test resourceVersion: "20096" uid: e95d1293-c581-4c0e-a069-57d660d758b5 kind: Event lastTimestamp: "2026-04-24T22:51:11Z" message: 'Created pod: sequence-graph-32c7c-6ff7f8b44c-rwdm4' metadata: creationTimestamp: "2026-04-24T22:51:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:51:11Z" name: sequence-graph-32c7c-6ff7f8b44c.18a96cb97f970f71 namespace: kserve-ci-e2e-test resourceVersion: "20100" uid: d1a20c73-cd85-4a5d-8732-e94f7ccca99b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-32c7c namespace: kserve-ci-e2e-test resourceVersion: "20094" uid: af7e148f-56dd-4c71-a10c-507c5371d6ff kind: Event lastTimestamp: "2026-04-24T22:51:11Z" message: Scaled up replica set sequence-graph-32c7c-6ff7f8b44c from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:51:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:51:11Z" name: sequence-graph-32c7c.18a96cb97f06426c namespace: kserve-ci-e2e-test resourceVersion: "20097" uid: 25df3378-b465-4e78-b4cb-cbbd016058ac reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-32c7c namespace: kserve-ci-e2e-test resourceVersion: "20113" uid: a36c9f57-0d10-452b-9567-f4e86d1faf74 kind: Event lastTimestamp: "2026-04-24T22:51:11Z" message: InferenceGraph [sequence-graph-32c7c] is Ready metadata: creationTimestamp: "2026-04-24T22:51:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:51:11Z" name: sequence-graph-32c7c.18a96cb982df2f16 namespace: kserve-ci-e2e-test resourceVersion: "20115" uid: ae9ccdf6-75c2-4ea1-9392-88b629df1b8c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:41:56.994823Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f8431-7f98d55687-f4dl5 namespace: kserve-ci-e2e-test resourceVersion: "15671" uid: c50b6da3-95f3-462c-96ea-3ccd90c93413 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-f8431-7f98d55687-f4dl5 to ip-10-0-133-161.ec2.internal metadata: creationTimestamp: "2026-04-24T22:41:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:41:56Z" name: sequence-graph-f8431-7f98d55687-f4dl5.18a96c384a4bff0d namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: ca6623b2-a3f1-4458-9043-88c7b5c4a444 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:57Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f8431-7f98d55687-f4dl5 namespace: kserve-ci-e2e-test resourceVersion: "15680" uid: c50b6da3-95f3-462c-96ea-3ccd90c93413 kind: Event lastTimestamp: "2026-04-24T22:41:57Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:41:57Z" name: sequence-graph-f8431-7f98d55687-f4dl5.18a96c3863329306 namespace: kserve-ci-e2e-test resourceVersion: "15696" uid: 88f5e0c6-5439-48ce-b8f3-de2ee8f722e5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f8431} kind: Pod name: sequence-graph-f8431-7f98d55687-f4dl5 namespace: kserve-ci-e2e-test resourceVersion: "15672" uid: c50b6da3-95f3-462c-96ea-3ccd90c93413 kind: Event lastTimestamp: "2026-04-24T22:41:57Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f3ef58ae57fd8b984890d9aeb7935958f0ea100fbddeb0833d76b7e2007120ff" already present on machine metadata: creationTimestamp: "2026-04-24T22:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:41:57Z" name: sequence-graph-f8431-7f98d55687-f4dl5.18a96c3864753b50 namespace: kserve-ci-e2e-test resourceVersion: "15700" uid: e6066ebf-4401-4579-afe6-0a169ab866a8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f8431} kind: Pod name: sequence-graph-f8431-7f98d55687-f4dl5 namespace: kserve-ci-e2e-test resourceVersion: "15672" uid: c50b6da3-95f3-462c-96ea-3ccd90c93413 kind: Event lastTimestamp: "2026-04-24T22:41:57Z" message: 'Created container: sequence-graph-f8431' metadata: creationTimestamp: "2026-04-24T22:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:41:57Z" name: sequence-graph-f8431-7f98d55687-f4dl5.18a96c386752176d namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: d1cfc325-ccf0-4db8-a2f8-69885f84ce7d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f8431} kind: Pod name: sequence-graph-f8431-7f98d55687-f4dl5 namespace: kserve-ci-e2e-test resourceVersion: "15672" uid: c50b6da3-95f3-462c-96ea-3ccd90c93413 kind: Event lastTimestamp: "2026-04-24T22:41:57Z" message: Started container sequence-graph-f8431 metadata: creationTimestamp: "2026-04-24T22:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:41:57Z" name: sequence-graph-f8431-7f98d55687-f4dl5.18a96c3867bb8a45 namespace: kserve-ci-e2e-test resourceVersion: "15703" uid: cf440b01-e4ea-47b9-836c-c38c361cac2f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f8431} kind: Pod name: sequence-graph-f8431-7f98d55687-f4dl5 namespace: kserve-ci-e2e-test resourceVersion: "15672" uid: c50b6da3-95f3-462c-96ea-3ccd90c93413 kind: Event lastTimestamp: "2026-04-24T22:50:11Z" message: Stopping container sequence-graph-f8431 metadata: creationTimestamp: "2026-04-24T22:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:11Z" name: sequence-graph-f8431-7f98d55687-f4dl5.18a96cab7ce413b2 namespace: kserve-ci-e2e-test resourceVersion: "19267" uid: f9875aec-e9c9-465e-9417-b5228939bd7a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f8431} kind: Pod name: sequence-graph-f8431-7f98d55687-f4dl5 namespace: kserve-ci-e2e-test resourceVersion: "15672" uid: c50b6da3-95f3-462c-96ea-3ccd90c93413 kind: Event lastTimestamp: "2026-04-24T22:50:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T22:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:39Z" name: sequence-graph-f8431-7f98d55687-f4dl5.18a96cac09cb3fda namespace: kserve-ci-e2e-test resourceVersion: "19669" uid: aaae548f-728e-4259-87ed-edba6230f75c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-f8431-7f98d55687 namespace: kserve-ci-e2e-test resourceVersion: "15668" uid: 21150675-6427-43a9-8c06-f76b752b2f81 kind: Event lastTimestamp: "2026-04-24T22:41:56Z" message: 'Created pod: sequence-graph-f8431-7f98d55687-f4dl5' metadata: creationTimestamp: "2026-04-24T22:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:56Z" name: sequence-graph-f8431-7f98d55687.18a96c3849f13028 namespace: kserve-ci-e2e-test resourceVersion: "15673" uid: e0bd12db-7230-4c85-b0bc-6901076aba0c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-f8431 namespace: kserve-ci-e2e-test resourceVersion: "15666" uid: 68d93593-e6e7-41b2-896f-6b93f758ad70 kind: Event lastTimestamp: "2026-04-24T22:41:56Z" message: Scaled up replica set sequence-graph-f8431-7f98d55687 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:56Z" name: sequence-graph-f8431.18a96c384906272d namespace: kserve-ci-e2e-test resourceVersion: "15669" uid: 233a2494-fe68-40a9-8f6c-25d2b7b5fe12 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:57Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-f8431 namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: d6aa75b1-b7c4-4402-9657-89300ecaa236 kind: Event lastTimestamp: "2026-04-24T22:41:57Z" message: InferenceGraph [sequence-graph-f8431] is Ready metadata: creationTimestamp: "2026-04-24T22:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:41:57Z" name: sequence-graph-f8431.18a96c384eb27ca3 namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: c876f303-211b-4941-9fc7-4d9455e8e8b9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:42:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f8431 namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: 31dcc4b2-983d-4ac0-96c2-fda97e9fb445 kind: Event lastTimestamp: "2026-04-24T22:42:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:42:27Z" name: sequence-graph-f8431.18a96c3bc7e0f183 namespace: kserve-ci-e2e-test resourceVersion: "15912" uid: 42dbd9d9-ecb5-4b28-8bd6-12eb13da43dc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:42:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f8431 namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: 31dcc4b2-983d-4ac0-96c2-fda97e9fb445 kind: Event lastTimestamp: "2026-04-24T22:42:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:42:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:42:27Z" name: sequence-graph-f8431.18a96c3bc82ab5e5 namespace: kserve-ci-e2e-test resourceVersion: "15913" uid: d13d3c74-d254-42eb-b846-84aca1cc1734 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T23:00:57.340314Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-44b1e-98dcdcd84-rr45w namespace: kserve-ci-e2e-test resourceVersion: "24847" uid: b0c9f0f2-6888-48a6-aec5-6cb581452bd7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-44b1e-98dcdcd84-rr45w to ip-10-0-133-161.ec2.internal metadata: creationTimestamp: "2026-04-24T23:00:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T23:00:57Z" name: splitter-graph-44b1e-98dcdcd84-rr45w.18a96d41cc2f3a6a namespace: kserve-ci-e2e-test resourceVersion: "24852" uid: aed91c03-1d8f-4de9-bec0-ea5296e1f06d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:57Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-44b1e-98dcdcd84-rr45w namespace: kserve-ci-e2e-test resourceVersion: "24849" uid: b0c9f0f2-6888-48a6-aec5-6cb581452bd7 kind: Event lastTimestamp: "2026-04-24T23:00:57Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-44b1e-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T23:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:57Z" name: splitter-graph-44b1e-98dcdcd84-rr45w.18a96d41d5e19173 namespace: kserve-ci-e2e-test resourceVersion: "24865" uid: ef804480-4708-4c94-9053-62950c74164a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:58Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-44b1e-98dcdcd84-rr45w namespace: kserve-ci-e2e-test resourceVersion: "24857" uid: b0c9f0f2-6888-48a6-aec5-6cb581452bd7 kind: Event lastTimestamp: "2026-04-24T23:00:58Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T23:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T23:00:58Z" name: splitter-graph-44b1e-98dcdcd84-rr45w.18a96d420947b39f namespace: kserve-ci-e2e-test resourceVersion: "24874" uid: cc95327e-47fd-4d5e-b167-43cf2d6214be reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-44b1e} kind: Pod name: splitter-graph-44b1e-98dcdcd84-rr45w namespace: kserve-ci-e2e-test resourceVersion: "24849" uid: b0c9f0f2-6888-48a6-aec5-6cb581452bd7 kind: Event lastTimestamp: "2026-04-24T23:00:58Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f3ef58ae57fd8b984890d9aeb7935958f0ea100fbddeb0833d76b7e2007120ff" already present on machine metadata: creationTimestamp: "2026-04-24T23:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:58Z" name: splitter-graph-44b1e-98dcdcd84-rr45w.18a96d420a7cde75 namespace: kserve-ci-e2e-test resourceVersion: "24876" uid: 20f397ae-2874-4a5d-ab36-8852a65bb682 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-44b1e} kind: Pod name: splitter-graph-44b1e-98dcdcd84-rr45w namespace: kserve-ci-e2e-test resourceVersion: "24849" uid: b0c9f0f2-6888-48a6-aec5-6cb581452bd7 kind: Event lastTimestamp: "2026-04-24T23:00:58Z" message: 'Created container: splitter-graph-44b1e' metadata: creationTimestamp: "2026-04-24T23:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:58Z" name: splitter-graph-44b1e-98dcdcd84-rr45w.18a96d420d4efd5b namespace: kserve-ci-e2e-test resourceVersion: "24878" uid: c28c3f93-a726-481f-b8bf-2647fc408811 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-44b1e} kind: Pod name: splitter-graph-44b1e-98dcdcd84-rr45w namespace: kserve-ci-e2e-test resourceVersion: "24849" uid: b0c9f0f2-6888-48a6-aec5-6cb581452bd7 kind: Event lastTimestamp: "2026-04-24T23:00:58Z" message: Started container splitter-graph-44b1e metadata: creationTimestamp: "2026-04-24T23:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:58Z" name: splitter-graph-44b1e-98dcdcd84-rr45w.18a96d420dbd0790 namespace: kserve-ci-e2e-test resourceVersion: "24879" uid: 852b40a9-0d1f-40a6-9207-ba3c46f1b26a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-44b1e} kind: Pod name: splitter-graph-44b1e-98dcdcd84-rr45w namespace: kserve-ci-e2e-test resourceVersion: "24849" uid: b0c9f0f2-6888-48a6-aec5-6cb581452bd7 kind: Event lastTimestamp: "2026-04-24T23:01:07Z" message: Stopping container splitter-graph-44b1e metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:07Z" name: splitter-graph-44b1e-98dcdcd84-rr45w.18a96d4425b09c34 namespace: kserve-ci-e2e-test resourceVersion: "24977" uid: f46b52d2-6d45-4434-afc2-ef7fec47b089 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T23:01:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-44b1e} kind: Pod name: splitter-graph-44b1e-98dcdcd84-rr45w namespace: kserve-ci-e2e-test resourceVersion: "24849" uid: b0c9f0f2-6888-48a6-aec5-6cb581452bd7 kind: Event lastTimestamp: "2026-04-24T23:01:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T23:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:34Z" name: splitter-graph-44b1e-98dcdcd84-rr45w.18a96d449f5f0533 namespace: kserve-ci-e2e-test resourceVersion: "25333" uid: 7e3238d7-8cd4-4c2a-ae03-c2fb38cc0b84 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-44b1e-98dcdcd84 namespace: kserve-ci-e2e-test resourceVersion: "24843" uid: b0f41b1c-dcfe-4dd0-a0d6-a550a3e7ea5d kind: Event lastTimestamp: "2026-04-24T23:00:57Z" message: 'Created pod: splitter-graph-44b1e-98dcdcd84-rr45w' metadata: creationTimestamp: "2026-04-24T23:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:00:57Z" name: splitter-graph-44b1e-98dcdcd84.18a96d41cbc650c3 namespace: kserve-ci-e2e-test resourceVersion: "24848" uid: bc95633e-6fdb-482b-a6b1-9b3f9b792cb9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-44b1e namespace: kserve-ci-e2e-test resourceVersion: "24842" uid: 893bd5ab-6dad-4fbe-8e71-9a30a142920e kind: Event lastTimestamp: "2026-04-24T23:00:57Z" message: Scaled up replica set splitter-graph-44b1e-98dcdcd84 from 0 to 1 metadata: creationTimestamp: "2026-04-24T23:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:00:57Z" name: splitter-graph-44b1e.18a96d41cb3164a3 namespace: kserve-ci-e2e-test resourceVersion: "24845" uid: f8898a9e-2062-4d3c-a676-f2bad4d43970 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:57Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-44b1e namespace: kserve-ci-e2e-test resourceVersion: "24860" uid: 0bf65a11-6fed-43a6-9692-a2afe3f96e0c kind: Event lastTimestamp: "2026-04-24T23:00:57Z" message: InferenceGraph [splitter-graph-44b1e] is Ready metadata: creationTimestamp: "2026-04-24T23:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:00:57Z" name: splitter-graph-44b1e.18a96d41cfc8bcd8 namespace: kserve-ci-e2e-test resourceVersion: "24862" uid: ad6189a5-daaf-4e4a-ab0a-b64548f6c48f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T23:02:07.613800Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-be7a5-64dbc87fdf-ddr2h namespace: kserve-ci-e2e-test resourceVersion: "25665" uid: bb88b2f1-c3bf-49c6-9641-c4dff86b9528 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-be7a5-64dbc87fdf-ddr2h to ip-10-0-133-161.ec2.internal metadata: creationTimestamp: "2026-04-24T23:02:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T23:02:07Z" name: splitter-graph-be7a5-64dbc87fdf-ddr2h.18a96d5228cf8993 namespace: kserve-ci-e2e-test resourceVersion: "25670" uid: d1757043-7ba0-41fb-9fb3-04380847ff6e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:02:08Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-be7a5-64dbc87fdf-ddr2h namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: bb88b2f1-c3bf-49c6-9641-c4dff86b9528 kind: Event lastTimestamp: "2026-04-24T23:02:08Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T23:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T23:02:08Z" name: splitter-graph-be7a5-64dbc87fdf-ddr2h.18a96d5241de1395 namespace: kserve-ci-e2e-test resourceVersion: "25688" uid: a5e6ef1f-8907-4736-b87d-4396cbad0b2f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:02:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-be7a5} kind: Pod name: splitter-graph-be7a5-64dbc87fdf-ddr2h namespace: kserve-ci-e2e-test resourceVersion: "25667" uid: bb88b2f1-c3bf-49c6-9641-c4dff86b9528 kind: Event lastTimestamp: "2026-04-24T23:02:08Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f3ef58ae57fd8b984890d9aeb7935958f0ea100fbddeb0833d76b7e2007120ff" already present on machine metadata: creationTimestamp: "2026-04-24T23:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:02:08Z" name: splitter-graph-be7a5-64dbc87fdf-ddr2h.18a96d5243072fd4 namespace: kserve-ci-e2e-test resourceVersion: "25690" uid: b3183a6d-9dc8-4440-b292-8f5f02d8497e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:02:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-be7a5} kind: Pod name: splitter-graph-be7a5-64dbc87fdf-ddr2h namespace: kserve-ci-e2e-test resourceVersion: "25667" uid: bb88b2f1-c3bf-49c6-9641-c4dff86b9528 kind: Event lastTimestamp: "2026-04-24T23:02:08Z" message: 'Created container: splitter-graph-be7a5' metadata: creationTimestamp: "2026-04-24T23:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:02:08Z" name: splitter-graph-be7a5-64dbc87fdf-ddr2h.18a96d5246c2b068 namespace: kserve-ci-e2e-test resourceVersion: "25691" uid: 245c085b-b844-4f82-bf9a-679fce16f312 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:02:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-be7a5} kind: Pod name: splitter-graph-be7a5-64dbc87fdf-ddr2h namespace: kserve-ci-e2e-test resourceVersion: "25667" uid: bb88b2f1-c3bf-49c6-9641-c4dff86b9528 kind: Event lastTimestamp: "2026-04-24T23:02:08Z" message: Started container splitter-graph-be7a5 metadata: creationTimestamp: "2026-04-24T23:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:02:08Z" name: splitter-graph-be7a5-64dbc87fdf-ddr2h.18a96d5247344973 namespace: kserve-ci-e2e-test resourceVersion: "25692" uid: 1b77d39b-3ce7-455d-a219-b2695afb3f45 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:10:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-be7a5} kind: Pod name: splitter-graph-be7a5-64dbc87fdf-ddr2h namespace: kserve-ci-e2e-test resourceVersion: "25667" uid: bb88b2f1-c3bf-49c6-9641-c4dff86b9528 kind: Event lastTimestamp: "2026-04-24T23:10:22Z" message: Stopping container splitter-graph-be7a5 metadata: creationTimestamp: "2026-04-24T23:10:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:10:22Z" name: splitter-graph-be7a5-64dbc87fdf-ddr2h.18a96dc553366da4 namespace: kserve-ci-e2e-test resourceVersion: "29410" uid: 53ca8a45-7d31-4697-99d8-fb88bf961744 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T23:10:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-be7a5} kind: Pod name: splitter-graph-be7a5-64dbc87fdf-ddr2h namespace: kserve-ci-e2e-test resourceVersion: "25667" uid: bb88b2f1-c3bf-49c6-9641-c4dff86b9528 kind: Event lastTimestamp: "2026-04-24T23:10:49Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T23:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:10:49Z" name: splitter-graph-be7a5-64dbc87fdf-ddr2h.18a96dc5e4dc06ab namespace: kserve-ci-e2e-test resourceVersion: "29644" uid: 1fd08452-7dd8-42f4-a586-1d4afbee3ab2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:02:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-be7a5-64dbc87fdf namespace: kserve-ci-e2e-test resourceVersion: "25661" uid: d3a6493c-5e2a-4dfb-b6bf-57719e6809e8 kind: Event lastTimestamp: "2026-04-24T23:02:07Z" message: 'Created pod: splitter-graph-be7a5-64dbc87fdf-ddr2h' metadata: creationTimestamp: "2026-04-24T23:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:02:07Z" name: splitter-graph-be7a5-64dbc87fdf.18a96d52286d129d namespace: kserve-ci-e2e-test resourceVersion: "25666" uid: 0feab1d6-655d-4236-bc2b-82b5a9206a85 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:02:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-be7a5 namespace: kserve-ci-e2e-test resourceVersion: "25658" uid: 72b7ec75-b1be-430d-8b1d-3830a917955b kind: Event lastTimestamp: "2026-04-24T23:02:07Z" message: Scaled up replica set splitter-graph-be7a5-64dbc87fdf from 0 to 1 metadata: creationTimestamp: "2026-04-24T23:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:02:07Z" name: splitter-graph-be7a5.18a96d5227b78a8a namespace: kserve-ci-e2e-test resourceVersion: "25663" uid: a129408e-b145-4ae7-aa0b-016b0999726a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:02:07Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-be7a5 namespace: kserve-ci-e2e-test resourceVersion: "25676" uid: 86ed25bc-8c99-42e7-85e3-54b1c49ffa48 kind: Event lastTimestamp: "2026-04-24T23:02:07Z" message: InferenceGraph [splitter-graph-be7a5] is Ready metadata: creationTimestamp: "2026-04-24T23:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:02:07Z" name: splitter-graph-be7a5.18a96d522bbed697 namespace: kserve-ci-e2e-test resourceVersion: "25678" uid: b7dff985-021c-4329-9d91-1af7b6ab2f54 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T23:02:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-be7a5 namespace: kserve-ci-e2e-test resourceVersion: "25660" uid: 3744a311-8829-4d1a-bbf8-23ec4b157ccf kind: Event lastTimestamp: "2026-04-24T23:02:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T23:02:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:02:52Z" name: splitter-graph-be7a5.18a96d55a6090fa1 namespace: kserve-ci-e2e-test resourceVersion: "26311" uid: cb2e8189-bcd5-47ed-9637-a9a26d4ea2c3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T23:02:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-be7a5 namespace: kserve-ci-e2e-test resourceVersion: "25660" uid: 3744a311-8829-4d1a-bbf8-23ec4b157ccf kind: Event lastTimestamp: "2026-04-24T23:02:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T23:02:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:02:52Z" name: splitter-graph-be7a5.18a96d55a668116a namespace: kserve-ci-e2e-test resourceVersion: "26312" uid: ec29d463-7ba7-4f7a-9cfe-133c6fb1c9aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:38:46.899584Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b namespace: kserve-ci-e2e-test resourceVersion: "13399" uid: 1239da72-897f-4238-97c9-d1096f1fea83 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1b9dc-predictor-68868569ff-tdd6b to ip-10-0-142-202.ec2.internal metadata: creationTimestamp: "2026-04-24T22:38:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:38:46Z" name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b.18a96c0c07bcbfb7 namespace: kserve-ci-e2e-test resourceVersion: "13404" uid: bae5ef69-9e76-412f-a205-04df807bfa05 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b namespace: kserve-ci-e2e-test resourceVersion: "13401" uid: 1239da72-897f-4238-97c9-d1096f1fea83 kind: Event lastTimestamp: "2026-04-24T22:38:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-1b9dc-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T22:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:47Z" name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b.18a96c0c0fee03c5 namespace: kserve-ci-e2e-test resourceVersion: "13426" uid: 6db09d54-ce09-4a6f-81f2-d096fd5573b9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b namespace: kserve-ci-e2e-test resourceVersion: "13407" uid: 1239da72-897f-4238-97c9-d1096f1fea83 kind: Event lastTimestamp: "2026-04-24T22:38:47Z" message: Add eth0 [10.132.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:38:47Z" name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b.18a96c0c449f164f namespace: kserve-ci-e2e-test resourceVersion: "13493" uid: 7f2e53b1-c1be-498a-abc1-26f4ac5d47a4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b namespace: kserve-ci-e2e-test resourceVersion: "13401" uid: 1239da72-897f-4238-97c9-d1096f1fea83 kind: Event lastTimestamp: "2026-04-24T22:38:47Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1443" metadata: creationTimestamp: "2026-04-24T22:38:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:38:47Z" name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b.18a96c0c45d95b2f namespace: kserve-ci-e2e-test resourceVersion: "13496" uid: e9db0c86-2e63-495d-a5ad-63d636edd0a8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b namespace: kserve-ci-e2e-test resourceVersion: "13401" uid: 1239da72-897f-4238-97c9-d1096f1fea83 kind: Event lastTimestamp: "2026-04-24T22:39:01Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1443" in 13.775s (13.775s including waiting). Image size: 1335781480 bytes.' metadata: creationTimestamp: "2026-04-24T22:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:01Z" name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b.18a96c0f7aec6d94 namespace: kserve-ci-e2e-test resourceVersion: "13662" uid: b90f79c7-6c3f-4756-bda8-f849a8cbdb2b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b namespace: kserve-ci-e2e-test resourceVersion: "13401" uid: 1239da72-897f-4238-97c9-d1096f1fea83 kind: Event lastTimestamp: "2026-04-24T22:39:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:01Z" name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b.18a96c0f7e772ad3 namespace: kserve-ci-e2e-test resourceVersion: "13663" uid: 2f942341-4d9d-41c1-ba08-dfadf80a8c25 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b namespace: kserve-ci-e2e-test resourceVersion: "13401" uid: 1239da72-897f-4238-97c9-d1096f1fea83 kind: Event lastTimestamp: "2026-04-24T22:39:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:01Z" name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b.18a96c0f7ef1ff38 namespace: kserve-ci-e2e-test resourceVersion: "13664" uid: 5f9afbd3-86fc-402d-ad90-19d26494a13b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b namespace: kserve-ci-e2e-test resourceVersion: "13401" uid: 1239da72-897f-4238-97c9-d1096f1fea83 kind: Event lastTimestamp: "2026-04-24T22:39:01Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-24T22:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:01Z" name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b.18a96c0f7f01e791 namespace: kserve-ci-e2e-test resourceVersion: "13665" uid: 0a5aa4c1-be81-4a21-90f6-c4e57e7b65ae reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b namespace: kserve-ci-e2e-test resourceVersion: "13401" uid: 1239da72-897f-4238-97c9-d1096f1fea83 kind: Event lastTimestamp: "2026-04-24T22:39:04Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.269s (2.269s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-24T22:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:04Z" name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b.18a96c100641cc3e namespace: kserve-ci-e2e-test resourceVersion: "13712" uid: 06a2a184-f068-4264-860d-8e09f200fca8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b namespace: kserve-ci-e2e-test resourceVersion: "13401" uid: 1239da72-897f-4238-97c9-d1096f1fea83 kind: Event lastTimestamp: "2026-04-24T22:39:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:04Z" name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b.18a96c1009ba023e namespace: kserve-ci-e2e-test resourceVersion: "13713" uid: 6b66a243-f1c4-4c09-bc90-c23b5278e3fd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b namespace: kserve-ci-e2e-test resourceVersion: "13401" uid: 1239da72-897f-4238-97c9-d1096f1fea83 kind: Event lastTimestamp: "2026-04-24T22:39:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:39:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:04Z" name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b.18a96c100a2cd2b6 namespace: kserve-ci-e2e-test resourceVersion: "13714" uid: 9d938db8-3bfc-458c-b711-14abd2c07f27 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:39:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b namespace: kserve-ci-e2e-test resourceVersion: "13401" uid: 1239da72-897f-4238-97c9-d1096f1fea83 kind: Event lastTimestamp: "2026-04-24T22:39:42Z" message: 'Readiness probe failed: dial tcp 10.132.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:39:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:39:42Z" name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b.18a96c1080e84111 namespace: kserve-ci-e2e-test resourceVersion: "14054" uid: 3d628768-b7eb-480e-8f2f-1b45f42caee9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b namespace: kserve-ci-e2e-test resourceVersion: "13401" uid: 1239da72-897f-4238-97c9-d1096f1fea83 kind: Event lastTimestamp: "2026-04-24T22:40:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:17Z" name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b.18a96c21047f5732 namespace: kserve-ci-e2e-test resourceVersion: "14447" uid: 13572f6b-39a4-412e-9dc7-6bd63947b66e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b namespace: kserve-ci-e2e-test resourceVersion: "13401" uid: 1239da72-897f-4238-97c9-d1096f1fea83 kind: Event lastTimestamp: "2026-04-24T22:40:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:17Z" name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b.18a96c21048050bf namespace: kserve-ci-e2e-test resourceVersion: "14451" uid: 20d2948c-05c1-44a5-bdcb-a0cf855fcd3c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b namespace: kserve-ci-e2e-test resourceVersion: "13401" uid: 1239da72-897f-4238-97c9-d1096f1fea83 kind: Event lastTimestamp: "2026-04-24T22:40:17Z" message: 'Readiness probe failed: Get "https://10.132.0.17:8643/healthz": dial tcp 10.132.0.17:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:17Z" name: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b.18a96c2108fc1a54 namespace: kserve-ci-e2e-test resourceVersion: "14474" uid: db912c24-ac6c-414c-b944-eab804317856 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1b9dc-predictor-68868569ff namespace: kserve-ci-e2e-test resourceVersion: "13393" uid: 54a9bdab-e47b-494c-925d-86c1cbc26219 kind: Event lastTimestamp: "2026-04-24T22:38:46Z" message: 'Created pod: success-200-isvc-1b9dc-predictor-68868569ff-tdd6b' metadata: creationTimestamp: "2026-04-24T22:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:38:46Z" name: success-200-isvc-1b9dc-predictor-68868569ff.18a96c0c0754e355 namespace: kserve-ci-e2e-test resourceVersion: "13402" uid: 09753f8b-cc5f-4e44-b1b4-b65b5f9cf6f0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:38:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1b9dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "13391" uid: f71a0356-1781-41f4-8bcf-5443d48841b4 kind: Event lastTimestamp: "2026-04-24T22:38:46Z" message: Scaled up replica set success-200-isvc-1b9dc-predictor-68868569ff from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:38:46Z" name: success-200-isvc-1b9dc-predictor.18a96c0c05a5c35c namespace: kserve-ci-e2e-test resourceVersion: "13396" uid: 8c328729-0829-4d39-b9bf-686572e7dec1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:39:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1b9dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "13392" uid: d6d7fc97-d482-4d25-bd7b-e0b418a1cb65 kind: Event lastTimestamp: "2026-04-24T22:39:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:39:31Z" name: success-200-isvc-1b9dc-predictor.18a96c0f84282032 namespace: kserve-ci-e2e-test resourceVersion: "13975" uid: adee7b2d-833f-4403-95db-4c5b4d9b4701 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:39:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1b9dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "13392" uid: d6d7fc97-d482-4d25-bd7b-e0b418a1cb65 kind: Event lastTimestamp: "2026-04-24T22:39:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:39:31Z" name: success-200-isvc-1b9dc-predictor.18a96c0f84a4d26c namespace: kserve-ci-e2e-test resourceVersion: "13976" uid: 7c5a0d6f-3b37-4ecf-98a4-a21a0db8756d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:39:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1b9dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "13672" uid: d6d7fc97-d482-4d25-bd7b-e0b418a1cb65 kind: Event lastTimestamp: "2026-04-24T22:40:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:16Z" name: success-200-isvc-1b9dc-predictor.18a96c1a00eb2041 namespace: kserve-ci-e2e-test resourceVersion: "14423" uid: c86f1161-e639-4db0-97b0-a110262f8647 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:39:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1b9dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "13672" uid: d6d7fc97-d482-4d25-bd7b-e0b418a1cb65 kind: Event lastTimestamp: "2026-04-24T22:40:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:16Z" name: success-200-isvc-1b9dc-predictor.18a96c1a01610e82 namespace: kserve-ci-e2e-test resourceVersion: "14425" uid: 9bec94c8-0e3c-4c14-915c-af57d9223f56 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:38:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1b9dc namespace: kserve-ci-e2e-test resourceVersion: "13383" uid: d6200c6c-7afc-4a25-995a-aa446e543653 kind: Event lastTimestamp: "2026-04-24T22:39:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:39:52Z" name: success-200-isvc-1b9dc.18a96c0c05774daf namespace: kserve-ci-e2e-test resourceVersion: "14161" uid: f55b5ee4-cf75-4974-9222-db40cf897624 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:39:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1b9dc namespace: kserve-ci-e2e-test resourceVersion: "14158" uid: d6200c6c-7afc-4a25-995a-aa446e543653 kind: Event lastTimestamp: "2026-04-24T22:39:52Z" message: InferenceService [success-200-isvc-1b9dc] is Ready metadata: creationTimestamp: "2026-04-24T22:39:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:39:52Z" name: success-200-isvc-1b9dc.18a96c1b3d71cc3c namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: 68f75304-c6b0-4bba-9304-d0de9b7ff226 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:51:22.180081Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t namespace: kserve-ci-e2e-test resourceVersion: "20266" uid: 7b0231fe-2a96-4d74-a5f8-9528010359db kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t to ip-10-0-142-202.ec2.internal metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:51:22Z" name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t.18a96cbbe1f68629 namespace: kserve-ci-e2e-test resourceVersion: "20274" uid: c424e07d-3397-4ce9-a37e-8f1eb30c2b73 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t namespace: kserve-ci-e2e-test resourceVersion: "20281" uid: 7b0231fe-2a96-4d74-a5f8-9528010359db kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:51:22Z" name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t.18a96cbbfb28e029 namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: 9d5b41e2-1bb2-47d9-91d3-44a074139b7a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 7b0231fe-2a96-4d74-a5f8-9528010359db kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:22Z" name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t.18a96cbbfc4d640a namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: 133e024b-367e-4ea1-98f2-75ac9fb3a994 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 7b0231fe-2a96-4d74-a5f8-9528010359db kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:22Z" name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t.18a96cbbfee7cd6f namespace: kserve-ci-e2e-test resourceVersion: "20326" uid: b663e60f-d302-4709-b18c-99c5636a7323 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 7b0231fe-2a96-4d74-a5f8-9528010359db kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:22Z" name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t.18a96cbbff691184 namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 9803e46b-b0cc-4861-8950-639ae55f3927 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 7b0231fe-2a96-4d74-a5f8-9528010359db kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:22Z" name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t.18a96cbbff755b18 namespace: kserve-ci-e2e-test resourceVersion: "20328" uid: 2cb39bc6-59b7-4602-813a-acab0b7cfa0b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 7b0231fe-2a96-4d74-a5f8-9528010359db kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:22Z" name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t.18a96cbc02916cf7 namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 55d683a7-b71c-4b0a-8265-eee44e06fc22 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 7b0231fe-2a96-4d74-a5f8-9528010359db kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:22Z" name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t.18a96cbc02fc469b namespace: kserve-ci-e2e-test resourceVersion: "20332" uid: 1d7de4cd-2073-47ff-8ac2-08f6f7741d02 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T22:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 7b0231fe-2a96-4d74-a5f8-9528010359db kind: Event lastTimestamp: "2026-04-24T23:00:40Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:40Z" name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t.18a96cbc589937cf namespace: kserve-ci-e2e-test resourceVersion: "24669" uid: 0fa374c7-ecf9-48a4-ac5f-bc55f3486fe8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 7b0231fe-2a96-4d74-a5f8-9528010359db kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:37Z" name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t.18a96d3d14e4c58e namespace: kserve-ci-e2e-test resourceVersion: "24555" uid: 0e2f6249-d9cd-4888-9839-1e6454fa9016 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 7b0231fe-2a96-4d74-a5f8-9528010359db kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:37Z" name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t.18a96d3d14e4ffde namespace: kserve-ci-e2e-test resourceVersion: "24558" uid: abd8cb43-29c7-4866-a8d8-f67284d07994 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: 7b0231fe-2a96-4d74-a5f8-9528010359db kind: Event lastTimestamp: "2026-04-24T23:00:40Z" message: 'Readiness probe failed: Get "https://10.132.0.24:8643/healthz": dial tcp 10.132.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T23:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:40Z" name: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t.18a96d3dcd18d5e1 namespace: kserve-ci-e2e-test resourceVersion: "24668" uid: 939b2c0b-4925-4765-9560-8e426cf3e768 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2e6fe-predictor-66c4d45c7c namespace: kserve-ci-e2e-test resourceVersion: "20253" uid: 8c6fa1f5-a4ee-4469-a4b0-83d6d5b41b14 kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: 'Created pod: success-200-isvc-2e6fe-predictor-66c4d45c7c-xmb5t' metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:51:22Z" name: success-200-isvc-2e6fe-predictor-66c4d45c7c.18a96cbbe19ce32d namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: ea14dfe0-a70b-4f74-9e56-b6a5e7add9a0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2e6fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20250" uid: d32b24a3-76fd-4f52-9e8e-52deb73efb23 kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: Scaled up replica set success-200-isvc-2e6fe-predictor-66c4d45c7c from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:51:22Z" name: success-200-isvc-2e6fe-predictor.18a96cbbdf9fdbf5 namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: ee877b60-3f83-40c9-95da-1fe1e9e321ff reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:51:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2e6fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20252" uid: f17ff88b-06ef-4a73-bd46-d88e61bc7f49 kind: Event lastTimestamp: "2026-04-24T22:52:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:51:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:52:07Z" name: success-200-isvc-2e6fe-predictor.18a96cbf5e3e463e namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: 416596fb-928f-4976-905a-4d997fd96d7b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:51:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2e6fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20252" uid: f17ff88b-06ef-4a73-bd46-d88e61bc7f49 kind: Event lastTimestamp: "2026-04-24T22:52:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:51:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:52:07Z" name: success-200-isvc-2e6fe-predictor.18a96cbf5ea3edc0 namespace: kserve-ci-e2e-test resourceVersion: "20773" uid: 45df7a2c-28db-4b9a-9542-fc294c1ebeb6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T22:51:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2e6fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: f17ff88b-06ef-4a73-bd46-d88e61bc7f49 kind: Event lastTimestamp: "2026-04-24T22:52:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:52:52Z" name: success-200-isvc-2e6fe-predictor.18a96cc2dd4aa44c namespace: kserve-ci-e2e-test resourceVersion: "21166" uid: ee7b58f0-f09a-4869-a030-58ecd018cb65 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T22:51:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2e6fe-predictor namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: f17ff88b-06ef-4a73-bd46-d88e61bc7f49 kind: Event lastTimestamp: "2026-04-24T22:52:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:52:52Z" name: success-200-isvc-2e6fe-predictor.18a96cc2dd916389 namespace: kserve-ci-e2e-test resourceVersion: "21168" uid: 5cf6de29-7567-4b23-bc59-e7efcacd2b88 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2e6fe namespace: kserve-ci-e2e-test resourceVersion: "20228" uid: df179046-e0dc-4bb1-9001-53bba704e14d kind: Event lastTimestamp: "2026-04-24T22:52:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:52:10Z" name: success-200-isvc-2e6fe.18a96cbbdf9ca4da namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: a3492d4f-e36d-4220-a105-205b210ea6bc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2e6fe namespace: kserve-ci-e2e-test resourceVersion: "20228" uid: df179046-e0dc-4bb1-9001-53bba704e14d kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: 'Failed to update status for InferenceService "success-200-isvc-2e6fe": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2e6fe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:51:22Z" name: success-200-isvc-2e6fe.18a96cbbe07bae4f namespace: kserve-ci-e2e-test resourceVersion: "20258" uid: 3ff1532e-b0bc-4a7f-a8ea-ecab815c4e84 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2e6fe namespace: kserve-ci-e2e-test resourceVersion: "20228" uid: df179046-e0dc-4bb1-9001-53bba704e14d kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2e6fe": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:51:22Z" name: success-200-isvc-2e6fe.18a96cbbe07c18e8 namespace: kserve-ci-e2e-test resourceVersion: "20260" uid: 5b9f76fc-175f-4f70-8918-ee0c146490df reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:52:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2e6fe namespace: kserve-ci-e2e-test resourceVersion: "20811" uid: df179046-e0dc-4bb1-9001-53bba704e14d kind: Event lastTimestamp: "2026-04-24T22:52:10Z" message: InferenceService [success-200-isvc-2e6fe] is Ready metadata: creationTimestamp: "2026-04-24T22:52:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:52:10Z" name: success-200-isvc-2e6fe.18a96cc713656ff9 namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: 6f427ef5-444c-4176-9c9d-951e7c080507 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:50:11.948743Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8 namespace: kserve-ci-e2e-test resourceVersion: "19304" uid: 4ed68bcb-5c74-4d05-a91c-593e9270ab6d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-32c7c-predictor-57df59874f-7vtc8 to ip-10-0-142-202.ec2.internal metadata: creationTimestamp: "2026-04-24T22:50:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:50:11Z" name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8.18a96cab87d96199 namespace: kserve-ci-e2e-test resourceVersion: "19309" uid: 76fc3877-7a66-447f-9b5b-12ed697a5503 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 4ed68bcb-5c74-4d05-a91c-593e9270ab6d kind: Event lastTimestamp: "2026-04-24T22:50:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-32c7c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T22:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:12Z" name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8.18a96cab97d9c6d8 namespace: kserve-ci-e2e-test resourceVersion: "19357" uid: d08adfb8-3d04-4937-b51f-f4c59075131e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8 namespace: kserve-ci-e2e-test resourceVersion: "19314" uid: 4ed68bcb-5c74-4d05-a91c-593e9270ab6d kind: Event lastTimestamp: "2026-04-24T22:50:12Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:50:12Z" name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8.18a96cabc4b2f2da namespace: kserve-ci-e2e-test resourceVersion: "19369" uid: 57d7c022-ec1f-4d18-8096-50d3a48acf82 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 4ed68bcb-5c74-4d05-a91c-593e9270ab6d kind: Event lastTimestamp: "2026-04-24T22:50:12Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:12Z" name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8.18a96cabc5f9c92a namespace: kserve-ci-e2e-test resourceVersion: "19372" uid: cf2616fe-0bde-4b8d-a600-f9856780b7e1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 4ed68bcb-5c74-4d05-a91c-593e9270ab6d kind: Event lastTimestamp: "2026-04-24T22:50:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:13Z" name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8.18a96cabc9b4bbdd namespace: kserve-ci-e2e-test resourceVersion: "19374" uid: 346dc132-e4f4-4620-afc5-3d9d4aecdfa2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 4ed68bcb-5c74-4d05-a91c-593e9270ab6d kind: Event lastTimestamp: "2026-04-24T22:50:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:13Z" name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8.18a96cabca25d8d9 namespace: kserve-ci-e2e-test resourceVersion: "19376" uid: 28b27ee0-4b34-4e1b-a295-84f59ebf0efd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 4ed68bcb-5c74-4d05-a91c-593e9270ab6d kind: Event lastTimestamp: "2026-04-24T22:50:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:13Z" name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8.18a96cabca33aa73 namespace: kserve-ci-e2e-test resourceVersion: "19377" uid: 5d2032be-7b11-447f-8c59-60c7b6f91924 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 4ed68bcb-5c74-4d05-a91c-593e9270ab6d kind: Event lastTimestamp: "2026-04-24T22:50:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:13Z" name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8.18a96cabcd1fd0b6 namespace: kserve-ci-e2e-test resourceVersion: "19381" uid: 2632f7a7-c174-4773-bea2-15ed5aa9b3cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 4ed68bcb-5c74-4d05-a91c-593e9270ab6d kind: Event lastTimestamp: "2026-04-24T22:50:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:13Z" name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8.18a96cabcd8ddd97 namespace: kserve-ci-e2e-test resourceVersion: "19382" uid: 6b49bae0-8c2d-4853-af27-950e1e8aca74 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 4ed68bcb-5c74-4d05-a91c-593e9270ab6d kind: Event lastTimestamp: "2026-04-24T22:50:50Z" message: 'Readiness probe failed: dial tcp 10.132.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:50Z" name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8.18a96cac3bbf15a0 namespace: kserve-ci-e2e-test resourceVersion: "19908" uid: bd29c870-37ec-41d7-8709-c2d73be5ca65 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 4ed68bcb-5c74-4d05-a91c-593e9270ab6d kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:22Z" name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8.18a96cbbde6810da namespace: kserve-ci-e2e-test resourceVersion: "20248" uid: 924c322b-0ca1-4bb3-8f0b-53af157d0f1d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8 namespace: kserve-ci-e2e-test resourceVersion: "19306" uid: 4ed68bcb-5c74-4d05-a91c-593e9270ab6d kind: Event lastTimestamp: "2026-04-24T22:51:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:51:22Z" name: success-200-isvc-32c7c-predictor-57df59874f-7vtc8.18a96cbbde69fc71 namespace: kserve-ci-e2e-test resourceVersion: "20251" uid: 13ae5211-496d-47f6-b1de-7e0613dafb32 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-32c7c-predictor-57df59874f namespace: kserve-ci-e2e-test resourceVersion: "19292" uid: 5195b457-2c22-4e5a-bdff-407a872467a2 kind: Event lastTimestamp: "2026-04-24T22:50:11Z" message: 'Created pod: success-200-isvc-32c7c-predictor-57df59874f-7vtc8' metadata: creationTimestamp: "2026-04-24T22:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:11Z" name: success-200-isvc-32c7c-predictor-57df59874f.18a96cab87879db5 namespace: kserve-ci-e2e-test resourceVersion: "19307" uid: 500b2a75-ab45-4da4-a4f5-f256716f477a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-32c7c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19286" uid: 27846d34-aa78-4a43-a24d-e85bb8332c53 kind: Event lastTimestamp: "2026-04-24T22:50:11Z" message: Scaled up replica set success-200-isvc-32c7c-predictor-57df59874f from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:11Z" name: success-200-isvc-32c7c-predictor.18a96cab8584a2e9 namespace: kserve-ci-e2e-test resourceVersion: "19294" uid: 5ec768b4-9988-4492-855f-d4bc3209d5f9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:50:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32c7c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19293" uid: 5cf5a9ff-fd4b-4e6f-8d84-b1de592d4d0c kind: Event lastTimestamp: "2026-04-24T22:50:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:56Z" name: success-200-isvc-32c7c-predictor.18a96caf0493d94e namespace: kserve-ci-e2e-test resourceVersion: "19946" uid: a5c4ccbb-1751-43f8-97af-e85fa258e8a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:50:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32c7c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19293" uid: 5cf5a9ff-fd4b-4e6f-8d84-b1de592d4d0c kind: Event lastTimestamp: "2026-04-24T22:50:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:56Z" name: success-200-isvc-32c7c-predictor.18a96caf04ff1a35 namespace: kserve-ci-e2e-test resourceVersion: "19947" uid: 59b1d2ed-667d-40ca-a8ae-a1bef38cc95b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32c7c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19527" uid: 5cf5a9ff-fd4b-4e6f-8d84-b1de592d4d0c kind: Event lastTimestamp: "2026-04-24T22:51:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:51:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:51:11Z" name: success-200-isvc-32c7c-predictor.18a96cb981e98b13 namespace: kserve-ci-e2e-test resourceVersion: "20109" uid: 679d6792-5686-4957-bffc-b1e2b3d2db4e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32c7c-predictor namespace: kserve-ci-e2e-test resourceVersion: "19527" uid: 5cf5a9ff-fd4b-4e6f-8d84-b1de592d4d0c kind: Event lastTimestamp: "2026-04-24T22:51:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:51:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:51:11Z" name: success-200-isvc-32c7c-predictor.18a96cb98248e808 namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: f0176ef9-7f81-40bc-83f5-c97c72139a33 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:50:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-32c7c namespace: kserve-ci-e2e-test resourceVersion: "19270" uid: 7ddf53ba-6f38-4e80-b01f-15311e2d9838 kind: Event lastTimestamp: "2026-04-24T22:51:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:51:01Z" name: success-200-isvc-32c7c.18a96cab85ca0c69 namespace: kserve-ci-e2e-test resourceVersion: "20005" uid: 0c8ec829-81cc-4136-b6b4-4d51902b65d5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-32c7c namespace: kserve-ci-e2e-test resourceVersion: "19270" uid: 7ddf53ba-6f38-4e80-b01f-15311e2d9838 kind: Event lastTimestamp: "2026-04-24T22:50:11Z" message: 'Failed to update status for InferenceService "success-200-isvc-32c7c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-32c7c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:50:11Z" name: success-200-isvc-32c7c.18a96cab8644a9f1 namespace: kserve-ci-e2e-test resourceVersion: "19299" uid: 9cb46a62-57d6-453b-b65b-ab2be79d407f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-32c7c namespace: kserve-ci-e2e-test resourceVersion: "19270" uid: 7ddf53ba-6f38-4e80-b01f-15311e2d9838 kind: Event lastTimestamp: "2026-04-24T22:50:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-32c7c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:50:11Z" name: success-200-isvc-32c7c.18a96cab8644f2ed namespace: kserve-ci-e2e-test resourceVersion: "19303" uid: 6afbe13b-b19a-4bd7-a53d-2814669cffe7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-32c7c namespace: kserve-ci-e2e-test resourceVersion: "20003" uid: 7ddf53ba-6f38-4e80-b01f-15311e2d9838 kind: Event lastTimestamp: "2026-04-24T22:51:01Z" message: InferenceService [success-200-isvc-32c7c] is Ready metadata: creationTimestamp: "2026-04-24T22:51:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:51:01Z" name: success-200-isvc-32c7c.18a96cb6f6b289f0 namespace: kserve-ci-e2e-test resourceVersion: "20004" uid: 53e18937-c7c2-4c33-abb0-c2532977544b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:59:57.277338Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-44b1e-predictor-589485df45-9jvsp namespace: kserve-ci-e2e-test resourceVersion: "24132" uid: ab2f9ce5-6e26-405a-809c-d2ab579d7913 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-44b1e-predictor-589485df45-9jvsp to ip-10-0-142-202.ec2.internal metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:59:57Z" name: success-200-isvc-44b1e-predictor-589485df45-9jvsp.18a96d33d026fbc9 namespace: kserve-ci-e2e-test resourceVersion: "24141" uid: b5750023-c51d-4beb-ae61-64481cf1bdab reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-44b1e-predictor-589485df45-9jvsp namespace: kserve-ci-e2e-test resourceVersion: "24136" uid: ab2f9ce5-6e26-405a-809c-d2ab579d7913 kind: Event lastTimestamp: "2026-04-24T22:59:57Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-44b1e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:59:57Z" name: success-200-isvc-44b1e-predictor-589485df45-9jvsp.18a96d33dc35d629 namespace: kserve-ci-e2e-test resourceVersion: "24169" uid: 2e2bdba1-39ff-480d-b7d1-fe5d2f14f806 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:58Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-44b1e-predictor-589485df45-9jvsp namespace: kserve-ci-e2e-test resourceVersion: "24151" uid: ab2f9ce5-6e26-405a-809c-d2ab579d7913 kind: Event lastTimestamp: "2026-04-24T22:59:58Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:59:58Z" name: success-200-isvc-44b1e-predictor-589485df45-9jvsp.18a96d340d1f1a98 namespace: kserve-ci-e2e-test resourceVersion: "24187" uid: 0bf433f1-0ef4-4b9c-b212-c08b4aa09eaa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-44b1e-predictor-589485df45-9jvsp namespace: kserve-ci-e2e-test resourceVersion: "24136" uid: ab2f9ce5-6e26-405a-809c-d2ab579d7913 kind: Event lastTimestamp: "2026-04-24T22:59:58Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:59:58Z" name: success-200-isvc-44b1e-predictor-589485df45-9jvsp.18a96d340e50b125 namespace: kserve-ci-e2e-test resourceVersion: "24189" uid: 1c0e4e7d-e9ad-4513-b0b7-b7ece9fd8e81 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-44b1e-predictor-589485df45-9jvsp namespace: kserve-ci-e2e-test resourceVersion: "24136" uid: ab2f9ce5-6e26-405a-809c-d2ab579d7913 kind: Event lastTimestamp: "2026-04-24T22:59:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:59:58Z" name: success-200-isvc-44b1e-predictor-589485df45-9jvsp.18a96d341152765c namespace: kserve-ci-e2e-test resourceVersion: "24191" uid: 272b5747-8315-4df9-9674-c093abc9b04c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-44b1e-predictor-589485df45-9jvsp namespace: kserve-ci-e2e-test resourceVersion: "24136" uid: ab2f9ce5-6e26-405a-809c-d2ab579d7913 kind: Event lastTimestamp: "2026-04-24T22:59:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:59:58Z" name: success-200-isvc-44b1e-predictor-589485df45-9jvsp.18a96d3411c30f84 namespace: kserve-ci-e2e-test resourceVersion: "24192" uid: 09b58d25-e132-48b4-9c8d-498eb11c4d9c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-44b1e-predictor-589485df45-9jvsp namespace: kserve-ci-e2e-test resourceVersion: "24136" uid: ab2f9ce5-6e26-405a-809c-d2ab579d7913 kind: Event lastTimestamp: "2026-04-24T22:59:58Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:59:58Z" name: success-200-isvc-44b1e-predictor-589485df45-9jvsp.18a96d3411d087e9 namespace: kserve-ci-e2e-test resourceVersion: "24193" uid: 3cedf4bb-c718-4ec9-9bfc-f81b069a85d9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-44b1e-predictor-589485df45-9jvsp namespace: kserve-ci-e2e-test resourceVersion: "24136" uid: ab2f9ce5-6e26-405a-809c-d2ab579d7913 kind: Event lastTimestamp: "2026-04-24T22:59:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:59:58Z" name: success-200-isvc-44b1e-predictor-589485df45-9jvsp.18a96d34163a0990 namespace: kserve-ci-e2e-test resourceVersion: "24194" uid: a06f9658-66ac-4d7c-8606-be644d50262a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-44b1e-predictor-589485df45-9jvsp namespace: kserve-ci-e2e-test resourceVersion: "24136" uid: ab2f9ce5-6e26-405a-809c-d2ab579d7913 kind: Event lastTimestamp: "2026-04-24T22:59:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:59:58Z" name: success-200-isvc-44b1e-predictor-589485df45-9jvsp.18a96d3416abad52 namespace: kserve-ci-e2e-test resourceVersion: "24195" uid: a41db45e-b8cd-4779-ab08-f3fd208c21ef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:59:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-44b1e-predictor-589485df45-9jvsp namespace: kserve-ci-e2e-test resourceVersion: "24136" uid: ab2f9ce5-6e26-405a-809c-d2ab579d7913 kind: Event lastTimestamp: "2026-04-24T23:00:34Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:59:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:34Z" name: success-200-isvc-44b1e-predictor-589485df45-9jvsp.18a96d341bcf8a37 namespace: kserve-ci-e2e-test resourceVersion: "24486" uid: 5064b007-afe2-4958-a8b6-dab17700c08e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-44b1e-predictor-589485df45-9jvsp namespace: kserve-ci-e2e-test resourceVersion: "24136" uid: ab2f9ce5-6e26-405a-809c-d2ab579d7913 kind: Event lastTimestamp: "2026-04-24T23:01:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:07Z" name: success-200-isvc-44b1e-predictor-589485df45-9jvsp.18a96d442ee4e246 namespace: kserve-ci-e2e-test resourceVersion: "25035" uid: 71f24642-8bfd-47f8-bde2-920bda053f6a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-44b1e-predictor-589485df45-9jvsp namespace: kserve-ci-e2e-test resourceVersion: "24136" uid: ab2f9ce5-6e26-405a-809c-d2ab579d7913 kind: Event lastTimestamp: "2026-04-24T23:01:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:07Z" name: success-200-isvc-44b1e-predictor-589485df45-9jvsp.18a96d442ee621c0 namespace: kserve-ci-e2e-test resourceVersion: "25037" uid: e64a4e63-b944-42ee-9c0c-05617db6462f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-44b1e-predictor-589485df45-9jvsp namespace: kserve-ci-e2e-test resourceVersion: "24136" uid: ab2f9ce5-6e26-405a-809c-d2ab579d7913 kind: Event lastTimestamp: "2026-04-24T23:01:09Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T23:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:09Z" name: success-200-isvc-44b1e-predictor-589485df45-9jvsp.18a96d44a3ec72be namespace: kserve-ci-e2e-test resourceVersion: "25112" uid: b5a810b8-55b5-4164-8433-52923ea20f1d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-44b1e-predictor-589485df45 namespace: kserve-ci-e2e-test resourceVersion: "24107" uid: 60c18178-1355-430b-966e-c1d98f52279e kind: Event lastTimestamp: "2026-04-24T22:59:57Z" message: 'Created pod: success-200-isvc-44b1e-predictor-589485df45-9jvsp' metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:59:57Z" name: success-200-isvc-44b1e-predictor-589485df45.18a96d33cfc88b1c namespace: kserve-ci-e2e-test resourceVersion: "24135" uid: 14d42126-45d5-4864-b4a6-9782d9f88fd1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-44b1e-predictor namespace: kserve-ci-e2e-test resourceVersion: "24105" uid: 9e7d2529-e0a1-4246-b593-0682e7e87b51 kind: Event lastTimestamp: "2026-04-24T22:59:57Z" message: Scaled up replica set success-200-isvc-44b1e-predictor-589485df45 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:59:57Z" name: success-200-isvc-44b1e-predictor.18a96d33cbedbbf5 namespace: kserve-ci-e2e-test resourceVersion: "24111" uid: 0040f741-839f-451b-8b9a-1e8035ecb3de reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T23:00:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-44b1e-predictor namespace: kserve-ci-e2e-test resourceVersion: "24106" uid: 03d4d151-ab36-4290-b8a1-dd12e3bc1f44 kind: Event lastTimestamp: "2026-04-24T23:00:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T23:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:00:57Z" name: success-200-isvc-44b1e-predictor.18a96d374ac8764e namespace: kserve-ci-e2e-test resourceVersion: "24833" uid: 687d6d2e-d6cc-4b4e-a7dc-a77537e18d97 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T23:00:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-44b1e-predictor namespace: kserve-ci-e2e-test resourceVersion: "24106" uid: 03d4d151-ab36-4290-b8a1-dd12e3bc1f44 kind: Event lastTimestamp: "2026-04-24T23:00:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T23:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:00:57Z" name: success-200-isvc-44b1e-predictor.18a96d374b2a48e4 namespace: kserve-ci-e2e-test resourceVersion: "24835" uid: 380d2c63-8978-464c-8887-523e6d939fbd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-44b1e namespace: kserve-ci-e2e-test resourceVersion: "24084" uid: 14816250-15d8-4428-8630-398fd6d5eaf5 kind: Event lastTimestamp: "2026-04-24T23:00:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:00:44Z" name: success-200-isvc-44b1e.18a96d33cbc714f9 namespace: kserve-ci-e2e-test resourceVersion: "24722" uid: e83c50e0-5dda-4abc-adc0-5bbdadad70bd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-44b1e namespace: kserve-ci-e2e-test resourceVersion: "24719" uid: 14816250-15d8-4428-8630-398fd6d5eaf5 kind: Event lastTimestamp: "2026-04-24T23:00:44Z" message: InferenceService [success-200-isvc-44b1e] is Ready metadata: creationTimestamp: "2026-04-24T23:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:00:44Z" name: success-200-isvc-44b1e.18a96d3ed63e5299 namespace: kserve-ci-e2e-test resourceVersion: "24720" uid: c003575a-f917-4621-a019-393a5aef40f7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:50:42.252149Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9 namespace: kserve-ci-e2e-test resourceVersion: "19745" uid: 47c13b48-ba40-4eb0-b3c4-1e09c63114a9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4aba3-predictor-6dcd794499-dzrk9 to ip-10-0-142-202.ec2.internal metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:50:42Z" name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9.18a96cb29612b109 namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 3deedc18-54a3-4223-8612-1a1486c31eaa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9 namespace: kserve-ci-e2e-test resourceVersion: "19760" uid: 47c13b48-ba40-4eb0-b3c4-1e09c63114a9 kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:50:42Z" name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9.18a96cb2aef0ab75 namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: 4b1e023d-5091-4996-8101-dda1e491da44 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9 namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: 47c13b48-ba40-4eb0-b3c4-1e09c63114a9 kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:42Z" name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9.18a96cb2b02c91c0 namespace: kserve-ci-e2e-test resourceVersion: "19807" uid: cef01747-510c-41e1-b1c7-81964a8ae014 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9 namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: 47c13b48-ba40-4eb0-b3c4-1e09c63114a9 kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:42Z" name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9.18a96cb2b2beaf2e namespace: kserve-ci-e2e-test resourceVersion: "19811" uid: f0524947-25a4-4024-8a30-5acc36f15cf1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9 namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: 47c13b48-ba40-4eb0-b3c4-1e09c63114a9 kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:42Z" name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9.18a96cb2b335553f namespace: kserve-ci-e2e-test resourceVersion: "19812" uid: e05cf98c-0cf1-448f-acfb-58fec0aa8e5f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9 namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: 47c13b48-ba40-4eb0-b3c4-1e09c63114a9 kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:42Z" name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9.18a96cb2b341df6f namespace: kserve-ci-e2e-test resourceVersion: "19813" uid: 79048fb4-8568-46fa-9575-92ebc1b8c76b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9 namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: 47c13b48-ba40-4eb0-b3c4-1e09c63114a9 kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:42Z" name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9.18a96cb2b63071e6 namespace: kserve-ci-e2e-test resourceVersion: "19814" uid: 4d713be4-f579-438c-aa76-a190e466abb7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9 namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: 47c13b48-ba40-4eb0-b3c4-1e09c63114a9 kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:42Z" name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9.18a96cb2b69acb4d namespace: kserve-ci-e2e-test resourceVersion: "19815" uid: 77ee5a2d-7234-40d7-a7f8-2d86449408cd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-24T22:50:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9 namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: 47c13b48-ba40-4eb0-b3c4-1e09c63114a9 kind: Event lastTimestamp: "2026-04-24T23:00:00Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:00Z" name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9.18a96cb30158c14d namespace: kserve-ci-e2e-test resourceVersion: "24217" uid: e4769a14-41c5-4751-aac0-2e5d37236e71 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9 namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: 47c13b48-ba40-4eb0-b3c4-1e09c63114a9 kind: Event lastTimestamp: "2026-04-24T22:59:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:59:57Z" name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9.18a96d33caea322d namespace: kserve-ci-e2e-test resourceVersion: "24101" uid: 576f2e58-1536-4586-89ab-0793fb69837a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:59:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9 namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: 47c13b48-ba40-4eb0-b3c4-1e09c63114a9 kind: Event lastTimestamp: "2026-04-24T22:59:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:59:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:59:57Z" name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9.18a96d33caeb534a namespace: kserve-ci-e2e-test resourceVersion: "24104" uid: 9d1e1c49-14b3-4f5a-b558-dd232e60110a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9 namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: 47c13b48-ba40-4eb0-b3c4-1e09c63114a9 kind: Event lastTimestamp: "2026-04-24T23:00:00Z" message: 'Readiness probe failed: Get "https://10.132.0.23:8643/healthz": dial tcp 10.132.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T23:00:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:00Z" name: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9.18a96d3475aac365 namespace: kserve-ci-e2e-test resourceVersion: "24216" uid: 71234213-2e2f-4ca3-ba02-29398c931745 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4aba3-predictor-6dcd794499 namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 1c92f1f2-9fa1-428d-8e57-95e17db83029 kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: 'Created pod: success-200-isvc-4aba3-predictor-6dcd794499-dzrk9' metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:42Z" name: success-200-isvc-4aba3-predictor-6dcd794499.18a96cb295a3e123 namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: 2cef0eb3-96cc-420f-8a39-be2cf2495f09 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4aba3-predictor namespace: kserve-ci-e2e-test resourceVersion: "19725" uid: 35269f16-e544-41c9-b473-f70b3fc50300 kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: Scaled up replica set success-200-isvc-4aba3-predictor-6dcd794499 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:42Z" name: success-200-isvc-4aba3-predictor.18a96cb293decdb9 namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: eb5332ec-069f-4857-8089-92c11277873c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:50:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4aba3-predictor namespace: kserve-ci-e2e-test resourceVersion: "19733" uid: 44080507-fcdf-46ba-b9e8-216578880772 kind: Event lastTimestamp: "2026-04-24T22:51:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:51:12Z" name: success-200-isvc-4aba3-predictor.18a96cb612b453d3 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: c9d0b281-5983-48af-9ed5-bfa49af112be reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:50:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4aba3-predictor namespace: kserve-ci-e2e-test resourceVersion: "19733" uid: 44080507-fcdf-46ba-b9e8-216578880772 kind: Event lastTimestamp: "2026-04-24T22:51:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:51:12Z" name: success-200-isvc-4aba3-predictor.18a96cb6131a1adb namespace: kserve-ci-e2e-test resourceVersion: "20123" uid: bffc908d-0563-4e06-8bdd-053de52780a3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:51:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4aba3-predictor namespace: kserve-ci-e2e-test resourceVersion: "19954" uid: 44080507-fcdf-46ba-b9e8-216578880772 kind: Event lastTimestamp: "2026-04-24T22:51:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:51:57Z" name: success-200-isvc-4aba3-predictor.18a96cbd1096940e namespace: kserve-ci-e2e-test resourceVersion: "20695" uid: bea1bf26-65a1-41d8-b661-bbfec6f09048 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:51:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4aba3-predictor namespace: kserve-ci-e2e-test resourceVersion: "19954" uid: 44080507-fcdf-46ba-b9e8-216578880772 kind: Event lastTimestamp: "2026-04-24T22:51:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:51:57Z" name: success-200-isvc-4aba3-predictor.18a96cbd10ff5da8 namespace: kserve-ci-e2e-test resourceVersion: "20696" uid: 851d1dea-411b-4b46-bef6-b301ed8bc890 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4aba3 namespace: kserve-ci-e2e-test resourceVersion: "19712" uid: d3b3f606-fa0f-4b19-b979-f37ffd693c90 kind: Event lastTimestamp: "2026-04-24T22:51:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:51:30Z" name: success-200-isvc-4aba3.18a96cb2940b4933 namespace: kserve-ci-e2e-test resourceVersion: "20432" uid: 29c7449c-bf3b-43ef-9ea6-55a39437dc80 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4aba3 namespace: kserve-ci-e2e-test resourceVersion: "19712" uid: d3b3f606-fa0f-4b19-b979-f37ffd693c90 kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: 'Failed to update status for InferenceService "success-200-isvc-4aba3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4aba3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:50:42Z" name: success-200-isvc-4aba3.18a96cb294689bef namespace: kserve-ci-e2e-test resourceVersion: "19739" uid: 7aebca7d-46ee-4a9b-852f-47317969a497 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4aba3 namespace: kserve-ci-e2e-test resourceVersion: "19712" uid: d3b3f606-fa0f-4b19-b979-f37ffd693c90 kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4aba3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:50:42Z" name: success-200-isvc-4aba3.18a96cb29468e304 namespace: kserve-ci-e2e-test resourceVersion: "19742" uid: f708c6e8-d2e0-4427-9b8c-c48da0a112ad reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:51:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4aba3 namespace: kserve-ci-e2e-test resourceVersion: "20430" uid: d3b3f606-fa0f-4b19-b979-f37ffd693c90 kind: Event lastTimestamp: "2026-04-24T22:51:30Z" message: InferenceService [success-200-isvc-4aba3] is Ready metadata: creationTimestamp: "2026-04-24T22:51:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:51:30Z" name: success-200-isvc-4aba3.18a96cbdbc335c8b namespace: kserve-ci-e2e-test resourceVersion: "20431" uid: a905d13a-0bcf-45f2-9d94-849cadad3bb3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:49:31.992095Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5c921-predictor-8548d846d5-287kf namespace: kserve-ci-e2e-test resourceVersion: "18873" uid: 0c54d696-b7e8-4f6a-9d10-5868f5f05732 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5c921-predictor-8548d846d5-287kf to ip-10-0-142-202.ec2.internal metadata: creationTimestamp: "2026-04-24T22:49:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:49:31Z" name: success-200-isvc-5c921-predictor-8548d846d5-287kf.18a96ca23a3f519f namespace: kserve-ci-e2e-test resourceVersion: "18880" uid: 5386fb95-667b-43ca-bafa-94d2a2e75e21 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5c921-predictor-8548d846d5-287kf namespace: kserve-ci-e2e-test resourceVersion: "18877" uid: 0c54d696-b7e8-4f6a-9d10-5868f5f05732 kind: Event lastTimestamp: "2026-04-24T22:49:32Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-5c921-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T22:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:32Z" name: success-200-isvc-5c921-predictor-8548d846d5-287kf.18a96ca24804e2ac namespace: kserve-ci-e2e-test resourceVersion: "18915" uid: 69230658-693e-4e15-ab73-824deff173f1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:33Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5c921-predictor-8548d846d5-287kf namespace: kserve-ci-e2e-test resourceVersion: "18890" uid: 0c54d696-b7e8-4f6a-9d10-5868f5f05732 kind: Event lastTimestamp: "2026-04-24T22:49:33Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:49:33Z" name: success-200-isvc-5c921-predictor-8548d846d5-287kf.18a96ca276de2af6 namespace: kserve-ci-e2e-test resourceVersion: "18932" uid: f3abe067-ecf0-4068-9b2c-c201caf1acc2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5c921-predictor-8548d846d5-287kf namespace: kserve-ci-e2e-test resourceVersion: "18877" uid: 0c54d696-b7e8-4f6a-9d10-5868f5f05732 kind: Event lastTimestamp: "2026-04-24T22:49:33Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:33Z" name: success-200-isvc-5c921-predictor-8548d846d5-287kf.18a96ca2780f12e3 namespace: kserve-ci-e2e-test resourceVersion: "18935" uid: e01f3bd6-053c-4c43-ad8d-c995a9f770f7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5c921-predictor-8548d846d5-287kf namespace: kserve-ci-e2e-test resourceVersion: "18877" uid: 0c54d696-b7e8-4f6a-9d10-5868f5f05732 kind: Event lastTimestamp: "2026-04-24T22:49:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:33Z" name: success-200-isvc-5c921-predictor-8548d846d5-287kf.18a96ca27abff5d2 namespace: kserve-ci-e2e-test resourceVersion: "18937" uid: b65b83d4-6e42-4c35-81a7-dbc35ffc3625 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5c921-predictor-8548d846d5-287kf namespace: kserve-ci-e2e-test resourceVersion: "18877" uid: 0c54d696-b7e8-4f6a-9d10-5868f5f05732 kind: Event lastTimestamp: "2026-04-24T22:49:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:33Z" name: success-200-isvc-5c921-predictor-8548d846d5-287kf.18a96ca27b339fb7 namespace: kserve-ci-e2e-test resourceVersion: "18938" uid: 163a86b9-1c4b-4403-89f2-112cc0f12f10 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5c921-predictor-8548d846d5-287kf namespace: kserve-ci-e2e-test resourceVersion: "18877" uid: 0c54d696-b7e8-4f6a-9d10-5868f5f05732 kind: Event lastTimestamp: "2026-04-24T22:49:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:33Z" name: success-200-isvc-5c921-predictor-8548d846d5-287kf.18a96ca27b40d284 namespace: kserve-ci-e2e-test resourceVersion: "18939" uid: 04fb1c44-f494-43d0-8060-817622855b60 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5c921-predictor-8548d846d5-287kf namespace: kserve-ci-e2e-test resourceVersion: "18877" uid: 0c54d696-b7e8-4f6a-9d10-5868f5f05732 kind: Event lastTimestamp: "2026-04-24T22:49:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:33Z" name: success-200-isvc-5c921-predictor-8548d846d5-287kf.18a96ca27ea73708 namespace: kserve-ci-e2e-test resourceVersion: "18941" uid: d11a4991-b315-403e-a3c9-6380a665c859 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5c921-predictor-8548d846d5-287kf namespace: kserve-ci-e2e-test resourceVersion: "18877" uid: 0c54d696-b7e8-4f6a-9d10-5868f5f05732 kind: Event lastTimestamp: "2026-04-24T22:49:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:33Z" name: success-200-isvc-5c921-predictor-8548d846d5-287kf.18a96ca27f1954ab namespace: kserve-ci-e2e-test resourceVersion: "18942" uid: 447012a2-bb18-4ba0-a986-267bf4e8fc4b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:49:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5c921-predictor-8548d846d5-287kf namespace: kserve-ci-e2e-test resourceVersion: "18877" uid: 0c54d696-b7e8-4f6a-9d10-5868f5f05732 kind: Event lastTimestamp: "2026-04-24T22:50:09Z" message: 'Readiness probe failed: dial tcp 10.132.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:09Z" name: success-200-isvc-5c921-predictor-8548d846d5-287kf.18a96ca2a8b3cf55 namespace: kserve-ci-e2e-test resourceVersion: "19232" uid: a0d2c4a6-b7e5-4507-a440-981a842eca68 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5c921-predictor-8548d846d5-287kf namespace: kserve-ci-e2e-test resourceVersion: "18877" uid: 0c54d696-b7e8-4f6a-9d10-5868f5f05732 kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:42Z" name: success-200-isvc-5c921-predictor-8548d846d5-287kf.18a96cb294b0ebbb namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: 8c575c25-35d4-42b6-b920-50f595ce4b35 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5c921-predictor-8548d846d5-287kf namespace: kserve-ci-e2e-test resourceVersion: "18877" uid: 0c54d696-b7e8-4f6a-9d10-5868f5f05732 kind: Event lastTimestamp: "2026-04-24T22:50:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:42Z" name: success-200-isvc-5c921-predictor-8548d846d5-287kf.18a96cb294b15617 namespace: kserve-ci-e2e-test resourceVersion: "19744" uid: a380c88a-9e7b-4a1e-8513-1c151aefccf4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5c921-predictor-8548d846d5-287kf namespace: kserve-ci-e2e-test resourceVersion: "18877" uid: 0c54d696-b7e8-4f6a-9d10-5868f5f05732 kind: Event lastTimestamp: "2026-04-24T22:50:44Z" message: 'Readiness probe failed: Get "https://10.132.0.21:8643/healthz": dial tcp 10.132.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:44Z" name: success-200-isvc-5c921-predictor-8548d846d5-287kf.18a96cb330be7675 namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: 1bc88033-d9c4-4ea8-8b75-a2b83c84dff2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5c921-predictor-8548d846d5 namespace: kserve-ci-e2e-test resourceVersion: "18851" uid: f97331f9-e554-4560-b0bc-22ecf2172c7d kind: Event lastTimestamp: "2026-04-24T22:49:31Z" message: 'Created pod: success-200-isvc-5c921-predictor-8548d846d5-287kf' metadata: creationTimestamp: "2026-04-24T22:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:49:31Z" name: success-200-isvc-5c921-predictor-8548d846d5.18a96ca239e34086 namespace: kserve-ci-e2e-test resourceVersion: "18875" uid: ef4704be-336c-4f1a-bbd9-58ad2cd3991c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5c921-predictor namespace: kserve-ci-e2e-test resourceVersion: "18849" uid: 9d86656c-d459-4661-90c9-7c0d35888e35 kind: Event lastTimestamp: "2026-04-24T22:49:31Z" message: Scaled up replica set success-200-isvc-5c921-predictor-8548d846d5 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:49:31Z" name: success-200-isvc-5c921-predictor.18a96ca236af9ccc namespace: kserve-ci-e2e-test resourceVersion: "18853" uid: ca01a4f3-8bb1-4f73-8df7-b2501931f92c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5c921-predictor namespace: kserve-ci-e2e-test resourceVersion: "18850" uid: 98755103-3a83-4bd0-b7f5-7beb8babd45d kind: Event lastTimestamp: "2026-04-24T22:49:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:49:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:49:46Z" name: success-200-isvc-5c921-predictor.18a96ca5b51c5342 namespace: kserve-ci-e2e-test resourceVersion: "19061" uid: 74da4a6f-4248-400b-af41-5d1428b572de reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5c921-predictor namespace: kserve-ci-e2e-test resourceVersion: "18850" uid: 98755103-3a83-4bd0-b7f5-7beb8babd45d kind: Event lastTimestamp: "2026-04-24T22:49:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:49:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:49:46Z" name: success-200-isvc-5c921-predictor.18a96ca5b56432e6 namespace: kserve-ci-e2e-test resourceVersion: "19063" uid: 820f7f9c-a276-4eda-b6d0-c847c9b11c4f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:50:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5c921-predictor namespace: kserve-ci-e2e-test resourceVersion: "19062" uid: 98755103-3a83-4bd0-b7f5-7beb8babd45d kind: Event lastTimestamp: "2026-04-24T22:50:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:50:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:31Z" name: success-200-isvc-5c921-predictor.18a96ca934283fa7 namespace: kserve-ci-e2e-test resourceVersion: "19570" uid: e02ba4d3-82f9-4365-88fd-3e56c79bce96 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:50:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5c921-predictor namespace: kserve-ci-e2e-test resourceVersion: "19062" uid: 98755103-3a83-4bd0-b7f5-7beb8babd45d kind: Event lastTimestamp: "2026-04-24T22:50:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:50:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:50:31Z" name: success-200-isvc-5c921-predictor.18a96ca934928875 namespace: kserve-ci-e2e-test resourceVersion: "19572" uid: 7198fc30-b7b2-4b02-89cf-86a8e393bd1c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:49:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5c921 namespace: kserve-ci-e2e-test resourceVersion: "18826" uid: b9d90761-b06d-465f-961a-5a8069632304 kind: Event lastTimestamp: "2026-04-24T22:50:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:50:19Z" name: success-200-isvc-5c921.18a96ca23688f118 namespace: kserve-ci-e2e-test resourceVersion: "19467" uid: bc2cfcb3-707d-47cb-b2a7-2f3ece258b7b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5c921 namespace: kserve-ci-e2e-test resourceVersion: "18826" uid: b9d90761-b06d-465f-961a-5a8069632304 kind: Event lastTimestamp: "2026-04-24T22:49:31Z" message: 'Failed to update status for InferenceService "success-200-isvc-5c921": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5c921": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:49:31Z" name: success-200-isvc-5c921.18a96ca23700916e namespace: kserve-ci-e2e-test resourceVersion: "18856" uid: 6d5c6187-a3cd-465d-a150-d24ffdff00f3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5c921 namespace: kserve-ci-e2e-test resourceVersion: "18826" uid: b9d90761-b06d-465f-961a-5a8069632304 kind: Event lastTimestamp: "2026-04-24T22:49:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5c921": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T22:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:49:31Z" name: success-200-isvc-5c921.18a96ca23700d9e3 namespace: kserve-ci-e2e-test resourceVersion: "18857" uid: 10f1980b-e32c-4566-b105-ca54f9d4702b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5c921 namespace: kserve-ci-e2e-test resourceVersion: "19465" uid: b9d90761-b06d-465f-961a-5a8069632304 kind: Event lastTimestamp: "2026-04-24T22:50:19Z" message: InferenceService [success-200-isvc-5c921] is Ready metadata: creationTimestamp: "2026-04-24T22:50:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:50:19Z" name: success-200-isvc-5c921.18a96cad635ac62e namespace: kserve-ci-e2e-test resourceVersion: "19466" uid: 7a225480-b373-4e2f-bea0-e4a313cd17d4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T23:01:07.567022Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-be7a5-predictor-6654cff85-qtt26 namespace: kserve-ci-e2e-test resourceVersion: "25019" uid: d3891870-cb3c-45aa-a614-aa2168d2fd00 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-be7a5-predictor-6654cff85-qtt26 to ip-10-0-142-202.ec2.internal metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T23:01:07Z" name: success-200-isvc-be7a5-predictor-6654cff85-qtt26.18a96d442dbe70de namespace: kserve-ci-e2e-test resourceVersion: "25027" uid: bea40686-f604-462b-9458-29052fc9cbc6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:07Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-be7a5-predictor-6654cff85-qtt26 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: d3891870-cb3c-45aa-a614-aa2168d2fd00 kind: Event lastTimestamp: "2026-04-24T23:01:07Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-be7a5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:07Z" name: success-200-isvc-be7a5-predictor-6654cff85-qtt26.18a96d44369231a0 namespace: kserve-ci-e2e-test resourceVersion: "25063" uid: 4f518d8e-a651-4fa5-84c1-0d3beea088ca reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:08Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-be7a5-predictor-6654cff85-qtt26 namespace: kserve-ci-e2e-test resourceVersion: "25034" uid: d3891870-cb3c-45aa-a614-aa2168d2fd00 kind: Event lastTimestamp: "2026-04-24T23:01:08Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T23:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T23:01:08Z" name: success-200-isvc-be7a5-predictor-6654cff85-qtt26.18a96d446ac618b7 namespace: kserve-ci-e2e-test resourceVersion: "25081" uid: cbf3f104-d57f-4673-965e-7d7fa6429c2a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-be7a5-predictor-6654cff85-qtt26 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: d3891870-cb3c-45aa-a614-aa2168d2fd00 kind: Event lastTimestamp: "2026-04-24T23:01:08Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T23:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:08Z" name: success-200-isvc-be7a5-predictor-6654cff85-qtt26.18a96d446c017add namespace: kserve-ci-e2e-test resourceVersion: "25083" uid: 386bc16a-2bad-4abe-9db3-8c6e354d13a1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-be7a5-predictor-6654cff85-qtt26 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: d3891870-cb3c-45aa-a614-aa2168d2fd00 kind: Event lastTimestamp: "2026-04-24T23:01:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T23:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:08Z" name: success-200-isvc-be7a5-predictor-6654cff85-qtt26.18a96d446ede2291 namespace: kserve-ci-e2e-test resourceVersion: "25085" uid: d19f85e1-de43-46a5-be86-b3861262c1de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-be7a5-predictor-6654cff85-qtt26 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: d3891870-cb3c-45aa-a614-aa2168d2fd00 kind: Event lastTimestamp: "2026-04-24T23:01:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T23:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:08Z" name: success-200-isvc-be7a5-predictor-6654cff85-qtt26.18a96d446f52944c namespace: kserve-ci-e2e-test resourceVersion: "25086" uid: a3e8a54e-8913-47e6-85dd-6a80fa99faf5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-be7a5-predictor-6654cff85-qtt26 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: d3891870-cb3c-45aa-a614-aa2168d2fd00 kind: Event lastTimestamp: "2026-04-24T23:01:08Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T23:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:08Z" name: success-200-isvc-be7a5-predictor-6654cff85-qtt26.18a96d446f5f1ae7 namespace: kserve-ci-e2e-test resourceVersion: "25087" uid: 5fec8232-f1f5-407f-9a63-a100646d0bb5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-be7a5-predictor-6654cff85-qtt26 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: d3891870-cb3c-45aa-a614-aa2168d2fd00 kind: Event lastTimestamp: "2026-04-24T23:01:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T23:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:08Z" name: success-200-isvc-be7a5-predictor-6654cff85-qtt26.18a96d4472325bab namespace: kserve-ci-e2e-test resourceVersion: "25091" uid: 90d40676-6b22-460f-a7b8-cc498fdf690e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-be7a5-predictor-6654cff85-qtt26 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: d3891870-cb3c-45aa-a614-aa2168d2fd00 kind: Event lastTimestamp: "2026-04-24T23:01:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T23:01:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:08Z" name: success-200-isvc-be7a5-predictor-6654cff85-qtt26.18a96d4472a353b7 namespace: kserve-ci-e2e-test resourceVersion: "25094" uid: c3e084a2-51c7-4947-b75c-2665bc44c3cc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T23:01:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-be7a5-predictor-6654cff85-qtt26 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: d3891870-cb3c-45aa-a614-aa2168d2fd00 kind: Event lastTimestamp: "2026-04-24T23:01:45Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T23:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:45Z" name: success-200-isvc-be7a5-predictor-6654cff85-qtt26.18a96d44b0101c30 namespace: kserve-ci-e2e-test resourceVersion: "25465" uid: daae290d-ee36-4bba-b5cb-3352434cfd27 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:10:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-be7a5-predictor-6654cff85-qtt26 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: d3891870-cb3c-45aa-a614-aa2168d2fd00 kind: Event lastTimestamp: "2026-04-24T23:10:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T23:10:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:10:22Z" name: success-200-isvc-be7a5-predictor-6654cff85-qtt26.18a96dc558a4ba59 namespace: kserve-ci-e2e-test resourceVersion: "29424" uid: e893cad6-2bb7-4a04-8820-3db43d6a9d4f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:10:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-be7a5-predictor-6654cff85-qtt26 namespace: kserve-ci-e2e-test resourceVersion: "25022" uid: d3891870-cb3c-45aa-a614-aa2168d2fd00 kind: Event lastTimestamp: "2026-04-24T23:10:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T23:10:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:10:22Z" name: success-200-isvc-be7a5-predictor-6654cff85-qtt26.18a96dc558a5fd9d namespace: kserve-ci-e2e-test resourceVersion: "29425" uid: 91390ba4-efd4-41f2-98d1-fb0e13ee43cd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-be7a5-predictor-6654cff85 namespace: kserve-ci-e2e-test resourceVersion: "25004" uid: f6c30158-b180-421c-87e0-f8fc47787a60 kind: Event lastTimestamp: "2026-04-24T23:01:07Z" message: 'Created pod: success-200-isvc-be7a5-predictor-6654cff85-qtt26' metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:01:07Z" name: success-200-isvc-be7a5-predictor-6654cff85.18a96d442d400549 namespace: kserve-ci-e2e-test resourceVersion: "25021" uid: 8ea6ce9c-a2a2-4047-8064-779139f1729d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-be7a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "24999" uid: a451b43c-6799-45f4-8b1d-980090ac0ca6 kind: Event lastTimestamp: "2026-04-24T23:01:07Z" message: Scaled up replica set success-200-isvc-be7a5-predictor-6654cff85 from 0 to 1 metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:01:07Z" name: success-200-isvc-be7a5-predictor.18a96d442b7d2dcb namespace: kserve-ci-e2e-test resourceVersion: "25007" uid: ff6eca77-bf70-4eb4-b0e5-057f86ebe130 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-be7a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "25003" uid: a0fd5d64-b43e-4f1e-9c34-902c71736fe3 kind: Event lastTimestamp: "2026-04-24T23:01:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T23:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:01:22Z" name: success-200-isvc-be7a5-predictor.18a96d47aa009fa5 namespace: kserve-ci-e2e-test resourceVersion: "25224" uid: 5b552d1c-ce8a-4292-ad0e-05c102c966d3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-be7a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "25003" uid: a0fd5d64-b43e-4f1e-9c34-902c71736fe3 kind: Event lastTimestamp: "2026-04-24T23:01:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T23:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:01:22Z" name: success-200-isvc-be7a5-predictor.18a96d47aa638259 namespace: kserve-ci-e2e-test resourceVersion: "25226" uid: 7456434e-e821-4912-b94c-e2e0aea556e2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T23:01:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-be7a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "25225" uid: a0fd5d64-b43e-4f1e-9c34-902c71736fe3 kind: Event lastTimestamp: "2026-04-24T23:02:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T23:01:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:02:22Z" name: success-200-isvc-be7a5-predictor.18a96d4b291b5f87 namespace: kserve-ci-e2e-test resourceVersion: "25800" uid: 9c7812f5-5a43-47b7-b04d-047af7e60e5a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T23:01:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-be7a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "25225" uid: a0fd5d64-b43e-4f1e-9c34-902c71736fe3 kind: Event lastTimestamp: "2026-04-24T23:02:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T23:01:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:02:22Z" name: success-200-isvc-be7a5-predictor.18a96d4b2984e3db namespace: kserve-ci-e2e-test resourceVersion: "25801" uid: a7cfffb4-3de9-4e66-a569-9f327a50a96a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T23:01:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-be7a5 namespace: kserve-ci-e2e-test resourceVersion: "24979" uid: a62a8cdb-d75e-46b4-b08d-aec968d69ee3 kind: Event lastTimestamp: "2026-04-24T23:01:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:01:55Z" name: success-200-isvc-be7a5.18a96d442b59b20a namespace: kserve-ci-e2e-test resourceVersion: "25557" uid: dff70c72-95f0-4878-8b9b-8c715c309c56 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-be7a5 namespace: kserve-ci-e2e-test resourceVersion: "24979" uid: a62a8cdb-d75e-46b4-b08d-aec968d69ee3 kind: Event lastTimestamp: "2026-04-24T23:01:07Z" message: 'Failed to update status for InferenceService "success-200-isvc-be7a5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-be7a5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:01:07Z" name: success-200-isvc-be7a5.18a96d442bc9f0eb namespace: kserve-ci-e2e-test resourceVersion: "25009" uid: 12e41d4b-96ea-4578-a2fe-da3b793dece9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-be7a5 namespace: kserve-ci-e2e-test resourceVersion: "24979" uid: a62a8cdb-d75e-46b4-b08d-aec968d69ee3 kind: Event lastTimestamp: "2026-04-24T23:01:07Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-be7a5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:01:07Z" name: success-200-isvc-be7a5.18a96d442bca5674 namespace: kserve-ci-e2e-test resourceVersion: "25011" uid: 24bfe8be-e149-49a4-a8bb-23ecb18817d8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-be7a5 namespace: kserve-ci-e2e-test resourceVersion: "25553" uid: a62a8cdb-d75e-46b4-b08d-aec968d69ee3 kind: Event lastTimestamp: "2026-04-24T23:01:55Z" message: InferenceService [success-200-isvc-be7a5] is Ready metadata: creationTimestamp: "2026-04-24T23:01:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:01:55Z" name: success-200-isvc-be7a5.18a96d4f6ae4f2be namespace: kserve-ci-e2e-test resourceVersion: "25556" uid: a26effbe-407b-487d-9b3e-d48656cb3192 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T23:00:37.123256Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5 namespace: kserve-ci-e2e-test resourceVersion: "24563" uid: 549d96f8-2628-49c9-b80f-f3fecb0ca7a9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5 to ip-10-0-142-202.ec2.internal metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T23:00:37Z" name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5.18a96d3d17276dcb namespace: kserve-ci-e2e-test resourceVersion: "24569" uid: 87260ddc-05cc-478a-82bc-0eb102489a67 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5 namespace: kserve-ci-e2e-test resourceVersion: "24585" uid: 549d96f8-2628-49c9-b80f-f3fecb0ca7a9 kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T23:00:37Z" name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5.18a96d3d3013f663 namespace: kserve-ci-e2e-test resourceVersion: "24619" uid: 57c915b3-921c-4ba3-8278-f1e8a2c462a2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5 namespace: kserve-ci-e2e-test resourceVersion: "24565" uid: 549d96f8-2628-49c9-b80f-f3fecb0ca7a9 kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:37Z" name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5.18a96d3d3146cb4c namespace: kserve-ci-e2e-test resourceVersion: "24622" uid: 4e5054ee-cf64-441c-93b8-4836fe6bb969 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5 namespace: kserve-ci-e2e-test resourceVersion: "24565" uid: 549d96f8-2628-49c9-b80f-f3fecb0ca7a9 kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:37Z" name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5.18a96d3d348683e2 namespace: kserve-ci-e2e-test resourceVersion: "24625" uid: 9a27d161-4e45-4a8b-87f5-8b3708543263 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5 namespace: kserve-ci-e2e-test resourceVersion: "24565" uid: 549d96f8-2628-49c9-b80f-f3fecb0ca7a9 kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:37Z" name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5.18a96d3d34f6dfde namespace: kserve-ci-e2e-test resourceVersion: "24626" uid: 0ec9eddc-ad31-4fd3-b4f0-a06f16ba3fa7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5 namespace: kserve-ci-e2e-test resourceVersion: "24565" uid: 549d96f8-2628-49c9-b80f-f3fecb0ca7a9 kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:37Z" name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5.18a96d3d3504979e namespace: kserve-ci-e2e-test resourceVersion: "24628" uid: 40bc34e7-62f2-49b9-a9b9-cad1985b937c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5 namespace: kserve-ci-e2e-test resourceVersion: "24565" uid: 549d96f8-2628-49c9-b80f-f3fecb0ca7a9 kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:37Z" name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5.18a96d3d37ef8273 namespace: kserve-ci-e2e-test resourceVersion: "24632" uid: f013aa84-f790-40fd-84cb-20ec67bfc63b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5 namespace: kserve-ci-e2e-test resourceVersion: "24565" uid: 549d96f8-2628-49c9-b80f-f3fecb0ca7a9 kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:00:37Z" name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5.18a96d3d3862d6e5 namespace: kserve-ci-e2e-test resourceVersion: "24633" uid: 7a7ec6ca-16c0-46e4-b0df-108456b44145 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T23:00:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5 namespace: kserve-ci-e2e-test resourceVersion: "24565" uid: 549d96f8-2628-49c9-b80f-f3fecb0ca7a9 kind: Event lastTimestamp: "2026-04-24T23:01:15Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T23:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:15Z" name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5.18a96d3daec6881a namespace: kserve-ci-e2e-test resourceVersion: "25166" uid: 22c9edff-e856-4e34-877d-ae7ab07ee017 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:17:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5 namespace: kserve-ci-e2e-test resourceVersion: "24565" uid: 549d96f8-2628-49c9-b80f-f3fecb0ca7a9 kind: Event lastTimestamp: "2026-04-24T23:17:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T23:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:17:56Z" name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5.18a96e2f28860c4f namespace: kserve-ci-e2e-test resourceVersion: "32583" uid: 6e3b36c4-1d82-4394-887c-2f5e6524c34c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:17:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5 namespace: kserve-ci-e2e-test resourceVersion: "24565" uid: 549d96f8-2628-49c9-b80f-f3fecb0ca7a9 kind: Event lastTimestamp: "2026-04-24T23:17:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T23:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:17:56Z" name: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5.18a96e2f28869613 namespace: kserve-ci-e2e-test resourceVersion: "32584" uid: b9baf544-7e25-4b23-b82a-279e974594ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d0532-predictor-6cfcdcc985 namespace: kserve-ci-e2e-test resourceVersion: "24553" uid: 1923afe9-786c-428c-ab89-1dd8beface2f kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: 'Created pod: success-200-isvc-d0532-predictor-6cfcdcc985-fjlp5' metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:00:37Z" name: success-200-isvc-d0532-predictor-6cfcdcc985.18a96d3d16c97439 namespace: kserve-ci-e2e-test resourceVersion: "24566" uid: 8478c3a2-fe31-4301-ab61-27d881651b12 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d0532-predictor namespace: kserve-ci-e2e-test resourceVersion: "24550" uid: 033b3e27-fa61-428e-a93a-ba485896e739 kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: Scaled up replica set success-200-isvc-d0532-predictor-6cfcdcc985 from 0 to 1 metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:00:37Z" name: success-200-isvc-d0532-predictor.18a96d3d14f769da namespace: kserve-ci-e2e-test resourceVersion: "24556" uid: 97d7d15d-5588-4cc5-b0eb-278f0f22e2b1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T23:00:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d0532-predictor namespace: kserve-ci-e2e-test resourceVersion: "24552" uid: f2dd433b-6d80-4926-a890-e0fdc199b713 kind: Event lastTimestamp: "2026-04-24T23:01:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T23:00:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:01:22Z" name: success-200-isvc-d0532-predictor.18a96d40937f1688 namespace: kserve-ci-e2e-test resourceVersion: "25215" uid: e6f3a654-b5fc-415f-9501-999daae566c5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T23:00:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d0532-predictor namespace: kserve-ci-e2e-test resourceVersion: "24552" uid: f2dd433b-6d80-4926-a890-e0fdc199b713 kind: Event lastTimestamp: "2026-04-24T23:01:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T23:00:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:01:22Z" name: success-200-isvc-d0532-predictor.18a96d4093e4b8e2 namespace: kserve-ci-e2e-test resourceVersion: "25216" uid: 0e13821b-6126-4f60-9840-e8abad45f10a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T23:01:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d0532-predictor namespace: kserve-ci-e2e-test resourceVersion: "24788" uid: f2dd433b-6d80-4926-a890-e0fdc199b713 kind: Event lastTimestamp: "2026-04-24T23:02:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:02:07Z" name: success-200-isvc-d0532-predictor.18a96d44129ef11d namespace: kserve-ci-e2e-test resourceVersion: "25641" uid: fafe9e55-5dd7-4807-9506-51a6592cb5f7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-24T23:01:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d0532-predictor namespace: kserve-ci-e2e-test resourceVersion: "24788" uid: f2dd433b-6d80-4926-a890-e0fdc199b713 kind: Event lastTimestamp: "2026-04-24T23:02:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T23:01:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:02:07Z" name: success-200-isvc-d0532-predictor.18a96d44130b8aad namespace: kserve-ci-e2e-test resourceVersion: "25643" uid: 04d49a05-564b-4a0e-bf00-34d1dc8dec61 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d0532 namespace: kserve-ci-e2e-test resourceVersion: "24529" uid: 7e08c9e0-b0ab-4ce1-bb88-96453fed2cff kind: Event lastTimestamp: "2026-04-24T23:01:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:01:25Z" name: success-200-isvc-d0532.18a96d3d14d28737 namespace: kserve-ci-e2e-test resourceVersion: "25269" uid: 0aa7f4a1-19a7-41ad-8a51-2b3983cba0ec reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d0532 namespace: kserve-ci-e2e-test resourceVersion: "24529" uid: 7e08c9e0-b0ab-4ce1-bb88-96453fed2cff kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: 'Failed to update status for InferenceService "success-200-isvc-d0532": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d0532": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:00:37Z" name: success-200-isvc-d0532.18a96d3d152edf03 namespace: kserve-ci-e2e-test resourceVersion: "24559" uid: 30beaa0e-785d-4963-9b06-a56443b82db3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:00:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d0532 namespace: kserve-ci-e2e-test resourceVersion: "24529" uid: 7e08c9e0-b0ab-4ce1-bb88-96453fed2cff kind: Event lastTimestamp: "2026-04-24T23:00:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d0532": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-24T23:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:00:37Z" name: success-200-isvc-d0532.18a96d3d152f3510 namespace: kserve-ci-e2e-test resourceVersion: "24560" uid: ec123e78-da27-473f-806b-e197d76a40f4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d0532 namespace: kserve-ci-e2e-test resourceVersion: "25265" uid: 7e08c9e0-b0ab-4ce1-bb88-96453fed2cff kind: Event lastTimestamp: "2026-04-24T23:01:25Z" message: InferenceService [success-200-isvc-d0532] is Ready metadata: creationTimestamp: "2026-04-24T23:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:01:25Z" name: success-200-isvc-d0532.18a96d486938fcbd namespace: kserve-ci-e2e-test resourceVersion: "25266" uid: 4404c205-4ca9-4775-ace7-39db4ebea891 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:40:17.156501Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq namespace: kserve-ci-e2e-test resourceVersion: "14475" uid: 538ad1fd-f113-4c14-aaab-910c458d0d9b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e938d-predictor-858f86b7bc-ckndq to ip-10-0-142-202.ec2.internal metadata: creationTimestamp: "2026-04-24T22:40:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:40:17Z" name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq.18a96c210b781201 namespace: kserve-ci-e2e-test resourceVersion: "14480" uid: 985c5210-4f24-40ca-bdbc-bd0ef8a5f734 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: 538ad1fd-f113-4c14-aaab-910c458d0d9b kind: Event lastTimestamp: "2026-04-24T22:40:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-e938d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T22:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:17Z" name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq.18a96c21187feacd namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: 3335555f-e156-47a8-9f94-70e593f0c258 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:18Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq namespace: kserve-ci-e2e-test resourceVersion: "14484" uid: 538ad1fd-f113-4c14-aaab-910c458d0d9b kind: Event lastTimestamp: "2026-04-24T22:40:18Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:40:18Z" name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq.18a96c214854b3c5 namespace: kserve-ci-e2e-test resourceVersion: "14530" uid: 46a356a9-fb7c-4077-b7da-3993b38046b2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: 538ad1fd-f113-4c14-aaab-910c458d0d9b kind: Event lastTimestamp: "2026-04-24T22:40:18Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:18Z" name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq.18a96c214984fe3d namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: cca7a59e-323a-46b6-b89c-f06cdf88442d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: 538ad1fd-f113-4c14-aaab-910c458d0d9b kind: Event lastTimestamp: "2026-04-24T22:40:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:18Z" name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq.18a96c214c8400e9 namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: ab26cdf8-a14c-4b3b-9d6c-1675e45ee33a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: 538ad1fd-f113-4c14-aaab-910c458d0d9b kind: Event lastTimestamp: "2026-04-24T22:40:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:18Z" name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq.18a96c214cfb554c namespace: kserve-ci-e2e-test resourceVersion: "14536" uid: 62e984c6-a1dc-4917-8a1b-7522bc76d13f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: 538ad1fd-f113-4c14-aaab-910c458d0d9b kind: Event lastTimestamp: "2026-04-24T22:40:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:18Z" name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq.18a96c214d0906c6 namespace: kserve-ci-e2e-test resourceVersion: "14537" uid: e608c948-81ae-49f4-bf04-704ca9802298 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: 538ad1fd-f113-4c14-aaab-910c458d0d9b kind: Event lastTimestamp: "2026-04-24T22:40:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:18Z" name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq.18a96c2150822a13 namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: 52fd8983-3815-43eb-b62a-f1ae0ba6f15c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: 538ad1fd-f113-4c14-aaab-910c458d0d9b kind: Event lastTimestamp: "2026-04-24T22:40:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:40:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:18Z" name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq.18a96c21510fd5aa namespace: kserve-ci-e2e-test resourceVersion: "14540" uid: 59428407-a9a7-4371-a950-b582107b0ba5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:40:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: 538ad1fd-f113-4c14-aaab-910c458d0d9b kind: Event lastTimestamp: "2026-04-24T22:40:56Z" message: 'Readiness probe failed: dial tcp 10.132.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:56Z" name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq.18a96c21c8f8da15 namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: f170155a-e9a1-4d66-b534-79a6b68f70df reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: 538ad1fd-f113-4c14-aaab-910c458d0d9b kind: Event lastTimestamp: "2026-04-24T22:49:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:31Z" name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq.18a96ca235aa95f4 namespace: kserve-ci-e2e-test resourceVersion: "18845" uid: df23ded3-eb6a-4fc9-b305-249b32f137de reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq namespace: kserve-ci-e2e-test resourceVersion: "14478" uid: 538ad1fd-f113-4c14-aaab-910c458d0d9b kind: Event lastTimestamp: "2026-04-24T22:49:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:31Z" name: success-200-isvc-e938d-predictor-858f86b7bc-ckndq.18a96ca235abc607 namespace: kserve-ci-e2e-test resourceVersion: "18848" uid: ba7d87fe-1117-463d-a62c-7ff0f0a8129f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e938d-predictor-858f86b7bc namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: 83ff06ad-6de4-4068-88b5-3a7d9da08bd1 kind: Event lastTimestamp: "2026-04-24T22:40:17Z" message: 'Created pod: success-200-isvc-e938d-predictor-858f86b7bc-ckndq' metadata: creationTimestamp: "2026-04-24T22:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:17Z" name: success-200-isvc-e938d-predictor-858f86b7bc.18a96c210a7d1fe5 namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 059af52f-6ea1-4f1b-af9c-a5d77f882a8a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e938d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14448" uid: 279b67fc-fcd9-4b11-93d1-8df190676422 kind: Event lastTimestamp: "2026-04-24T22:40:17Z" message: Scaled up replica set success-200-isvc-e938d-predictor-858f86b7bc from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:17Z" name: success-200-isvc-e938d-predictor.18a96c2106359b2e namespace: kserve-ci-e2e-test resourceVersion: "14457" uid: 2634bb64-6229-4031-9158-de351add7eaf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:40:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e938d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14452" uid: 69ca67a7-51d4-4006-891c-4182557a3e77 kind: Event lastTimestamp: "2026-04-24T22:41:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:02Z" name: success-200-isvc-e938d-predictor.18a96c2484b468ef namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: e4231f9a-4481-40a0-b4be-1c9b788ee54a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:40:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e938d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14452" uid: 69ca67a7-51d4-4006-891c-4182557a3e77 kind: Event lastTimestamp: "2026-04-24T22:41:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:02Z" name: success-200-isvc-e938d-predictor.18a96c248523259f namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: f73cec53-352e-4444-a97a-f4ee96f1a0de reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:41:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e938d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14689" uid: 69ca67a7-51d4-4006-891c-4182557a3e77 kind: Event lastTimestamp: "2026-04-24T22:41:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:41:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:32Z" name: success-200-isvc-e938d-predictor.18a96c2f01909b00 namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 87d92dac-e474-4136-988a-23be3a72ad85 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T22:41:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e938d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14689" uid: 69ca67a7-51d4-4006-891c-4182557a3e77 kind: Event lastTimestamp: "2026-04-24T22:41:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:41:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:32Z" name: success-200-isvc-e938d-predictor.18a96c2f020aa8a9 namespace: kserve-ci-e2e-test resourceVersion: "15456" uid: b0cd4acb-a829-41fb-a526-94a4ebe04997 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:40:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e938d namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: e694a103-7b97-40f5-97ea-a30ca3051aa6 kind: Event lastTimestamp: "2026-04-24T22:41:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:41:06Z" name: success-200-isvc-e938d.18a96c2106054976 namespace: kserve-ci-e2e-test resourceVersion: "15205" uid: 25a76d93-d7bc-4b84-8727-1eb7828d5dc2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e938d namespace: kserve-ci-e2e-test resourceVersion: "15202" uid: e694a103-7b97-40f5-97ea-a30ca3051aa6 kind: Event lastTimestamp: "2026-04-24T22:41:06Z" message: InferenceService [success-200-isvc-e938d] is Ready metadata: creationTimestamp: "2026-04-24T22:41:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:41:06Z" name: success-200-isvc-e938d.18a96c2c84ad148d namespace: kserve-ci-e2e-test resourceVersion: "15203" uid: 62e88c6e-36a8-46fa-8365-f74ac173d67b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:40:56.985774Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz namespace: kserve-ci-e2e-test resourceVersion: "15009" uid: 9234db16-92ec-497c-9e99-7f02307e6e28 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz to ip-10-0-142-202.ec2.internal metadata: creationTimestamp: "2026-04-24T22:40:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:40:56Z" name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz.18a96c2a517a8dce namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: bc453be0-8efd-4aab-a6c5-8df3202de4a0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz namespace: kserve-ci-e2e-test resourceVersion: "15031" uid: 9234db16-92ec-497c-9e99-7f02307e6e28 kind: Event lastTimestamp: "2026-04-24T22:40:57Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:40:57Z" name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz.18a96c2a6a937aea namespace: kserve-ci-e2e-test resourceVersion: "15066" uid: cf20e305-e6d1-4178-99eb-c17772a6ae5d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 9234db16-92ec-497c-9e99-7f02307e6e28 kind: Event lastTimestamp: "2026-04-24T22:40:57Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1443" already present on machine metadata: creationTimestamp: "2026-04-24T22:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:57Z" name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz.18a96c2a6bd01f48 namespace: kserve-ci-e2e-test resourceVersion: "15070" uid: 04dff138-8bb6-4d7b-be08-a586642fccc3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 9234db16-92ec-497c-9e99-7f02307e6e28 kind: Event lastTimestamp: "2026-04-24T22:40:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-24T22:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:57Z" name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz.18a96c2a6f379936 namespace: kserve-ci-e2e-test resourceVersion: "15073" uid: ab564571-94e5-4e2b-905d-59c632d26594 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 9234db16-92ec-497c-9e99-7f02307e6e28 kind: Event lastTimestamp: "2026-04-24T22:40:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-24T22:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:57Z" name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz.18a96c2a6fa681f4 namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: e0427c56-6afc-4232-8ea3-a95b2807e8e8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 9234db16-92ec-497c-9e99-7f02307e6e28 kind: Event lastTimestamp: "2026-04-24T22:40:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-24T22:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:57Z" name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz.18a96c2a6fb22e20 namespace: kserve-ci-e2e-test resourceVersion: "15076" uid: ac05bf9b-c855-4d81-910c-9f95527397ef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 9234db16-92ec-497c-9e99-7f02307e6e28 kind: Event lastTimestamp: "2026-04-24T22:40:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-24T22:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:57Z" name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz.18a96c2a72486e42 namespace: kserve-ci-e2e-test resourceVersion: "15077" uid: 4c5176ce-65ec-4d38-800c-93ac02d19061 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 9234db16-92ec-497c-9e99-7f02307e6e28 kind: Event lastTimestamp: "2026-04-24T22:40:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:57Z" name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz.18a96c2a72fff168 namespace: kserve-ci-e2e-test resourceVersion: "15078" uid: e6555098-8358-4351-8d6f-32a3c6f4d569 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:40:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 9234db16-92ec-497c-9e99-7f02307e6e28 kind: Event lastTimestamp: "2026-04-24T22:41:35Z" message: 'Readiness probe failed: dial tcp 10.132.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-24T22:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:41:35Z" name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz.18a96c2ae4dea04d namespace: kserve-ci-e2e-test resourceVersion: "15482" uid: fafe9b2c-21d8-4c54-984f-230e1504f6ca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 9234db16-92ec-497c-9e99-7f02307e6e28 kind: Event lastTimestamp: "2026-04-24T22:50:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-24T22:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:11Z" name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz.18a96cab865be54f namespace: kserve-ci-e2e-test resourceVersion: "19301" uid: 3d508019-41d3-488b-8a2a-7efd64a2e416 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:50:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz namespace: kserve-ci-e2e-test resourceVersion: "15014" uid: 9234db16-92ec-497c-9e99-7f02307e6e28 kind: Event lastTimestamp: "2026-04-24T22:50:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-24T22:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:11Z" name: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz.18a96cab865c5659 namespace: kserve-ci-e2e-test resourceVersion: "19302" uid: 6ba0fa74-49cc-432f-abdd-b334cdab5d02 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-202.ec2.internal source: component: kubelet host: ip-10-0-142-202.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f8431-predictor-6bcffc9d57 namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: b779ee20-db56-4c4e-bb49-9be59d4dc1ef kind: Event lastTimestamp: "2026-04-24T22:40:56Z" message: 'Created pod: success-200-isvc-f8431-predictor-6bcffc9d57-nq2nz' metadata: creationTimestamp: "2026-04-24T22:40:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:56Z" name: success-200-isvc-f8431-predictor-6bcffc9d57.18a96c2a50f55138 namespace: kserve-ci-e2e-test resourceVersion: "15013" uid: 93d98c7b-e3b6-40bb-b29f-989c210f5a21 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f8431-predictor namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: 30e2f395-6b4f-4aba-971c-f23d359f21f6 kind: Event lastTimestamp: "2026-04-24T22:40:56Z" message: Scaled up replica set success-200-isvc-f8431-predictor-6bcffc9d57 from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:40:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:56Z" name: success-200-isvc-f8431-predictor.18a96c2a4f0bc6fe namespace: kserve-ci-e2e-test resourceVersion: "14998" uid: f0f90e55-2864-4b15-bf19-46f13ab9ad31 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:41:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f8431-predictor namespace: kserve-ci-e2e-test resourceVersion: "14994" uid: 87a92093-102a-4f2b-a388-d78310488127 kind: Event lastTimestamp: "2026-04-24T22:41:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:41:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:57Z" name: success-200-isvc-f8431-predictor.18a96c2dcdecc13d namespace: kserve-ci-e2e-test resourceVersion: "15681" uid: 19eccec0-9ef3-429e-a6ea-d40bda66995f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-24T22:41:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f8431-predictor namespace: kserve-ci-e2e-test resourceVersion: "14994" uid: 87a92093-102a-4f2b-a388-d78310488127 kind: Event lastTimestamp: "2026-04-24T22:41:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:41:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:57Z" name: success-200-isvc-f8431-predictor.18a96c2dce5896fa namespace: kserve-ci-e2e-test resourceVersion: "15682" uid: 8f1c05e9-8f4d-48a1-977a-cd8864ad5838 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f8431-predictor namespace: kserve-ci-e2e-test resourceVersion: "15246" uid: 87a92093-102a-4f2b-a388-d78310488127 kind: Event lastTimestamp: "2026-04-24T22:41:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:41:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:26Z" name: success-200-isvc-f8431-predictor.18a96c314d2b6e97 namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 9ab25008-d457-4ccf-8957-4f4bc6c40445 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f8431-predictor namespace: kserve-ci-e2e-test resourceVersion: "15246" uid: 87a92093-102a-4f2b-a388-d78310488127 kind: Event lastTimestamp: "2026-04-24T22:41:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:41:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:26Z" name: success-200-isvc-f8431-predictor.18a96c314d979177 namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 851d95bc-fa88-4ac0-b22b-50d5394d757c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T22:40:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f8431 namespace: kserve-ci-e2e-test resourceVersion: "14963" uid: b3b5e88e-49dd-4ab5-b3c2-cd7405c3078b kind: Event lastTimestamp: "2026-04-24T22:41:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-24T22:40:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:41:45Z" name: success-200-isvc-f8431.18a96c2a4ee8eb6e namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: 246bb2f1-ca79-4b76-ad95-374a5b34b43c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f8431 namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: b3b5e88e-49dd-4ab5-b3c2-cd7405c3078b kind: Event lastTimestamp: "2026-04-24T22:41:45Z" message: InferenceService [success-200-isvc-f8431] is Ready metadata: creationTimestamp: "2026-04-24T22:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:41:45Z" name: success-200-isvc-f8431.18a96c359fba817b namespace: kserve-ci-e2e-test resourceVersion: "15567" uid: f74dee54-23d8-472e-8c2c-5c9cbdf614c3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:40:06.757725Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1b9dc-5995d747dd-q5dbh namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: 9f1af98a-c88d-49bb-af7e-c226b9119b89 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-1b9dc-5995d747dd-q5dbh to ip-10-0-133-161.ec2.internal metadata: creationTimestamp: "2026-04-24T22:40:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:40:06Z" name: switch-graph-1b9dc-5995d747dd-q5dbh.18a96c1e9fa74e88 namespace: kserve-ci-e2e-test resourceVersion: "14291" uid: 196aae0d-96c5-40ed-9466-46f92c202874 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:06Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1b9dc-5995d747dd-q5dbh namespace: kserve-ci-e2e-test resourceVersion: "14288" uid: 9f1af98a-c88d-49bb-af7e-c226b9119b89 kind: Event lastTimestamp: "2026-04-24T22:40:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-1b9dc-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T22:40:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:06Z" name: switch-graph-1b9dc-5995d747dd-q5dbh.18a96c1ea963f25f namespace: kserve-ci-e2e-test resourceVersion: "14302" uid: 33af60cf-3ab2-4580-8a51-2b7015fd7cbd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:07Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-1b9dc-5995d747dd-q5dbh namespace: kserve-ci-e2e-test resourceVersion: "14294" uid: 9f1af98a-c88d-49bb-af7e-c226b9119b89 kind: Event lastTimestamp: "2026-04-24T22:40:07Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:40:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:40:07Z" name: switch-graph-1b9dc-5995d747dd-q5dbh.18a96c1edc72511c namespace: kserve-ci-e2e-test resourceVersion: "14311" uid: b6b6ddbd-a119-488f-b50b-9fa415ba914c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1b9dc} kind: Pod name: switch-graph-1b9dc-5995d747dd-q5dbh namespace: kserve-ci-e2e-test resourceVersion: "14288" uid: 9f1af98a-c88d-49bb-af7e-c226b9119b89 kind: Event lastTimestamp: "2026-04-24T22:40:07Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:f3ef58ae57fd8b984890d9aeb7935958f0ea100fbddeb0833d76b7e2007120ff" metadata: creationTimestamp: "2026-04-24T22:40:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:07Z" name: switch-graph-1b9dc-5995d747dd-q5dbh.18a96c1eddc4226e namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: 5340b61d-24b9-4171-8795-fae44aacc798 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1b9dc} kind: Pod name: switch-graph-1b9dc-5995d747dd-q5dbh namespace: kserve-ci-e2e-test resourceVersion: "14288" uid: 9f1af98a-c88d-49bb-af7e-c226b9119b89 kind: Event lastTimestamp: "2026-04-24T22:40:09Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:f3ef58ae57fd8b984890d9aeb7935958f0ea100fbddeb0833d76b7e2007120ff" in 1.807s (1.807s including waiting). Image size: 216346192 bytes.' metadata: creationTimestamp: "2026-04-24T22:40:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:09Z" name: switch-graph-1b9dc-5995d747dd-q5dbh.18a96c1f497bb535 namespace: kserve-ci-e2e-test resourceVersion: "14328" uid: 1a0b4bcc-e654-4d6e-a900-43d0f6a92bb5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1b9dc} kind: Pod name: switch-graph-1b9dc-5995d747dd-q5dbh namespace: kserve-ci-e2e-test resourceVersion: "14288" uid: 9f1af98a-c88d-49bb-af7e-c226b9119b89 kind: Event lastTimestamp: "2026-04-24T22:40:09Z" message: 'Created container: switch-graph-1b9dc' metadata: creationTimestamp: "2026-04-24T22:40:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:09Z" name: switch-graph-1b9dc-5995d747dd-q5dbh.18a96c1f4ca64cac namespace: kserve-ci-e2e-test resourceVersion: "14330" uid: fc19a495-4395-426d-beab-b896e1f5e2f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1b9dc} kind: Pod name: switch-graph-1b9dc-5995d747dd-q5dbh namespace: kserve-ci-e2e-test resourceVersion: "14288" uid: 9f1af98a-c88d-49bb-af7e-c226b9119b89 kind: Event lastTimestamp: "2026-04-24T22:40:09Z" message: Started container switch-graph-1b9dc metadata: creationTimestamp: "2026-04-24T22:40:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:09Z" name: switch-graph-1b9dc-5995d747dd-q5dbh.18a96c1f4d5e2761 namespace: kserve-ci-e2e-test resourceVersion: "14331" uid: aa5e13d4-5d7b-4e25-a168-5a8a6ce6655b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1b9dc} kind: Pod name: switch-graph-1b9dc-5995d747dd-q5dbh namespace: kserve-ci-e2e-test resourceVersion: "14288" uid: 9f1af98a-c88d-49bb-af7e-c226b9119b89 kind: Event lastTimestamp: "2026-04-24T22:40:16Z" message: Stopping container switch-graph-1b9dc metadata: creationTimestamp: "2026-04-24T22:40:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:16Z" name: switch-graph-1b9dc-5995d747dd-q5dbh.18a96c20fdd49429 namespace: kserve-ci-e2e-test resourceVersion: "14421" uid: 68a54fa4-60d6-4b55-a02e-67eacfa3a5cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:40:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-1b9dc} kind: Pod name: switch-graph-1b9dc-5995d747dd-q5dbh namespace: kserve-ci-e2e-test resourceVersion: "14288" uid: 9f1af98a-c88d-49bb-af7e-c226b9119b89 kind: Event lastTimestamp: "2026-04-24T22:40:45Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T22:40:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:40:45Z" name: switch-graph-1b9dc-5995d747dd-q5dbh.18a96c21e398225c namespace: kserve-ci-e2e-test resourceVersion: "14801" uid: f6bf9e8d-734f-4dcf-bca0-49cf74dd1af5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-1b9dc-5995d747dd namespace: kserve-ci-e2e-test resourceVersion: "14283" uid: 04e582cd-137f-48b4-a543-2c9fd5852aca kind: Event lastTimestamp: "2026-04-24T22:40:06Z" message: 'Created pod: switch-graph-1b9dc-5995d747dd-q5dbh' metadata: creationTimestamp: "2026-04-24T22:40:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:06Z" name: switch-graph-1b9dc-5995d747dd.18a96c1e9f445756 namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 6b51ae7d-828b-406a-9a10-598bb09afc87 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-1b9dc namespace: kserve-ci-e2e-test resourceVersion: "14281" uid: 22a5e203-30a3-40ed-8959-e97393c0906f kind: Event lastTimestamp: "2026-04-24T22:40:06Z" message: Scaled up replica set switch-graph-1b9dc-5995d747dd from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:40:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:40:06Z" name: switch-graph-1b9dc.18a96c1e9e70afe7 namespace: kserve-ci-e2e-test resourceVersion: "14284" uid: 6dedff97-458b-4fd3-bde6-a41fe115712b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:40:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-1b9dc namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: c7ea95b1-8cb8-45da-99ec-6f75adec14cc kind: Event lastTimestamp: "2026-04-24T22:40:06Z" message: InferenceGraph [switch-graph-1b9dc] is Ready metadata: creationTimestamp: "2026-04-24T22:40:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:40:06Z" name: switch-graph-1b9dc.18a96c1ea3afa84e namespace: kserve-ci-e2e-test resourceVersion: "14300" uid: 41841d99-57ac-425d-8a45-e56290d10c1e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-24T23:01:37.190902Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d0532-75644c6c8f-ccgwj namespace: kserve-ci-e2e-test resourceVersion: "25370" uid: 88cfa972-2a64-4953-8cee-2b55422bc533 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-d0532-75644c6c8f-ccgwj to ip-10-0-133-161.ec2.internal metadata: creationTimestamp: "2026-04-24T23:01:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T23:01:37Z" name: switch-graph-d0532-75644c6c8f-ccgwj.18a96d4b1376ecba namespace: kserve-ci-e2e-test resourceVersion: "25376" uid: 96bd3f9b-cd13-432b-8452-3a4662e07b73 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:37Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d0532-75644c6c8f-ccgwj namespace: kserve-ci-e2e-test resourceVersion: "25373" uid: 88cfa972-2a64-4953-8cee-2b55422bc533 kind: Event lastTimestamp: "2026-04-24T23:01:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-d0532-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T23:01:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:37Z" name: switch-graph-d0532-75644c6c8f-ccgwj.18a96d4b1bef5f12 namespace: kserve-ci-e2e-test resourceVersion: "25389" uid: b43cd127-85d2-49e0-b1e5-638145dca82c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:38Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d0532-75644c6c8f-ccgwj namespace: kserve-ci-e2e-test resourceVersion: "25379" uid: 88cfa972-2a64-4953-8cee-2b55422bc533 kind: Event lastTimestamp: "2026-04-24T23:01:38Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T23:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T23:01:38Z" name: switch-graph-d0532-75644c6c8f-ccgwj.18a96d4b4ff6a544 namespace: kserve-ci-e2e-test resourceVersion: "25404" uid: 5a8b66aa-c0e0-4a50-afce-0c770ea73a9f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d0532} kind: Pod name: switch-graph-d0532-75644c6c8f-ccgwj namespace: kserve-ci-e2e-test resourceVersion: "25373" uid: 88cfa972-2a64-4953-8cee-2b55422bc533 kind: Event lastTimestamp: "2026-04-24T23:01:38Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f3ef58ae57fd8b984890d9aeb7935958f0ea100fbddeb0833d76b7e2007120ff" already present on machine metadata: creationTimestamp: "2026-04-24T23:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:38Z" name: switch-graph-d0532-75644c6c8f-ccgwj.18a96d4b5123cf92 namespace: kserve-ci-e2e-test resourceVersion: "25406" uid: 94797533-c263-489c-a904-c499125c165a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d0532} kind: Pod name: switch-graph-d0532-75644c6c8f-ccgwj namespace: kserve-ci-e2e-test resourceVersion: "25373" uid: 88cfa972-2a64-4953-8cee-2b55422bc533 kind: Event lastTimestamp: "2026-04-24T23:01:38Z" message: 'Created container: switch-graph-d0532' metadata: creationTimestamp: "2026-04-24T23:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:38Z" name: switch-graph-d0532-75644c6c8f-ccgwj.18a96d4b545737cb namespace: kserve-ci-e2e-test resourceVersion: "25408" uid: 01a6251d-5388-4509-9b89-6b9a32d4cdb9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d0532} kind: Pod name: switch-graph-d0532-75644c6c8f-ccgwj namespace: kserve-ci-e2e-test resourceVersion: "25373" uid: 88cfa972-2a64-4953-8cee-2b55422bc533 kind: Event lastTimestamp: "2026-04-24T23:01:38Z" message: Started container switch-graph-d0532 metadata: creationTimestamp: "2026-04-24T23:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:01:38Z" name: switch-graph-d0532-75644c6c8f-ccgwj.18a96d4b54cd55cd namespace: kserve-ci-e2e-test resourceVersion: "25409" uid: 1c4cd699-045e-4a16-beee-dda2bf197870 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:17:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d0532} kind: Pod name: switch-graph-d0532-75644c6c8f-ccgwj namespace: kserve-ci-e2e-test resourceVersion: "25373" uid: 88cfa972-2a64-4953-8cee-2b55422bc533 kind: Event lastTimestamp: "2026-04-24T23:17:56Z" message: Stopping container switch-graph-d0532 metadata: creationTimestamp: "2026-04-24T23:17:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:17:56Z" name: switch-graph-d0532-75644c6c8f-ccgwj.18a96e2f219d61ba namespace: kserve-ci-e2e-test resourceVersion: "32570" uid: f7d864c1-dbcb-4f51-9d62-500abf3e227e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-24T23:17:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d0532} kind: Pod name: switch-graph-d0532-75644c6c8f-ccgwj namespace: kserve-ci-e2e-test resourceVersion: "25373" uid: 88cfa972-2a64-4953-8cee-2b55422bc533 kind: Event lastTimestamp: "2026-04-24T23:18:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T23:17:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T23:18:19Z" name: switch-graph-d0532-75644c6c8f-ccgwj.18a96e2fcfd0f555 namespace: kserve-ci-e2e-test resourceVersion: "32802" uid: 81e33b45-99fc-46b3-8c35-d92f9db340de reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-d0532-75644c6c8f namespace: kserve-ci-e2e-test resourceVersion: "25367" uid: add30630-d37b-4de2-b50b-fe219aab3c44 kind: Event lastTimestamp: "2026-04-24T23:01:37Z" message: 'Created pod: switch-graph-d0532-75644c6c8f-ccgwj' metadata: creationTimestamp: "2026-04-24T23:01:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:01:37Z" name: switch-graph-d0532-75644c6c8f.18a96d4b13144789 namespace: kserve-ci-e2e-test resourceVersion: "25372" uid: f89c8498-a781-4b52-b7b4-f166faaa26e0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-d0532 namespace: kserve-ci-e2e-test resourceVersion: "25363" uid: c6defb53-64a6-4536-951e-2c153220eb3f kind: Event lastTimestamp: "2026-04-24T23:01:37Z" message: Scaled up replica set switch-graph-d0532-75644c6c8f from 0 to 1 metadata: creationTimestamp: "2026-04-24T23:01:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:01:37Z" name: switch-graph-d0532.18a96d4b1284aa3a namespace: kserve-ci-e2e-test resourceVersion: "25368" uid: b29e5fde-3bc0-46bd-a85a-7d6c8c60c0ff reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T23:01:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-d0532 namespace: kserve-ci-e2e-test resourceVersion: "25382" uid: 7753e0a3-0cda-4861-9121-0dc06a82cc75 kind: Event lastTimestamp: "2026-04-24T23:01:37Z" message: InferenceGraph [switch-graph-d0532] is Ready metadata: creationTimestamp: "2026-04-24T23:01:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T23:01:37Z" name: switch-graph-d0532.18a96d4b1650fba5 namespace: kserve-ci-e2e-test resourceVersion: "25384" uid: ac7652f4-62a8-437f-abbf-82773063c875 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T23:01:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d0532 namespace: kserve-ci-e2e-test resourceVersion: "25366" uid: 27ecd16a-2b0b-4077-91f1-706a4ee7b738 kind: Event lastTimestamp: "2026-04-24T23:02:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T23:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:02:07Z" name: switch-graph-d0532.18a96d4e90f37181 namespace: kserve-ci-e2e-test resourceVersion: "25640" uid: 8f4bc161-a85f-41d4-af0b-d21bfbff5a71 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-24T23:01:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d0532 namespace: kserve-ci-e2e-test resourceVersion: "25366" uid: 27ecd16a-2b0b-4077-91f1-706a4ee7b738 kind: Event lastTimestamp: "2026-04-24T23:02:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T23:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T23:02:07Z" name: switch-graph-d0532.18a96d4e91376efd namespace: kserve-ci-e2e-test resourceVersion: "25642" uid: 6c74e128-2987-4fde-916b-9c76f86aa81c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-24T22:41:17.182086Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e938d-798f566f4f-4lmhz namespace: kserve-ci-e2e-test resourceVersion: "15308" uid: d92fa55c-f6a7-4353-b1d9-a8935574e0fd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-e938d-798f566f4f-4lmhz to ip-10-0-133-161.ec2.internal metadata: creationTimestamp: "2026-04-24T22:41:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-24T22:41:17Z" name: switch-graph-e938d-798f566f4f-4lmhz.18a96c2f0545cb9c namespace: kserve-ci-e2e-test resourceVersion: "15313" uid: 111b5857-a11f-42ed-a7a1-0a791c27f287 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5688845f4-xnbbb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:17Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e938d-798f566f4f-4lmhz namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: d92fa55c-f6a7-4353-b1d9-a8935574e0fd kind: Event lastTimestamp: "2026-04-24T22:41:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-e938d-serving-cert" not found' metadata: creationTimestamp: "2026-04-24T22:41:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:41:17Z" name: switch-graph-e938d-798f566f4f-4lmhz.18a96c2f148f12bf namespace: kserve-ci-e2e-test resourceVersion: "15327" uid: d237d01a-94cd-41fc-8206-feab49e972b9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:18Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e938d-798f566f4f-4lmhz namespace: kserve-ci-e2e-test resourceVersion: "15317" uid: d92fa55c-f6a7-4353-b1d9-a8935574e0fd kind: Event lastTimestamp: "2026-04-24T22:41:18Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-24T22:41:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-24T22:41:18Z" name: switch-graph-e938d-798f566f4f-4lmhz.18a96c2f42026266 namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: 4d9958e9-40b1-479c-86a8-0ce767e96597 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e938d} kind: Pod name: switch-graph-e938d-798f566f4f-4lmhz namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: d92fa55c-f6a7-4353-b1d9-a8935574e0fd kind: Event lastTimestamp: "2026-04-24T22:41:18Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:f3ef58ae57fd8b984890d9aeb7935958f0ea100fbddeb0833d76b7e2007120ff" already present on machine metadata: creationTimestamp: "2026-04-24T22:41:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:41:18Z" name: switch-graph-e938d-798f566f4f-4lmhz.18a96c2f43489585 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 9ab8ab75-7884-43be-8c0c-7e3264e46062 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e938d} kind: Pod name: switch-graph-e938d-798f566f4f-4lmhz namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: d92fa55c-f6a7-4353-b1d9-a8935574e0fd kind: Event lastTimestamp: "2026-04-24T22:41:18Z" message: 'Created container: switch-graph-e938d' metadata: creationTimestamp: "2026-04-24T22:41:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:41:18Z" name: switch-graph-e938d-798f566f4f-4lmhz.18a96c2f460788c8 namespace: kserve-ci-e2e-test resourceVersion: "15338" uid: faa1fbed-f4ac-4ebf-8844-75b2b32d9cd1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e938d} kind: Pod name: switch-graph-e938d-798f566f4f-4lmhz namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: d92fa55c-f6a7-4353-b1d9-a8935574e0fd kind: Event lastTimestamp: "2026-04-24T22:41:18Z" message: Started container switch-graph-e938d metadata: creationTimestamp: "2026-04-24T22:41:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:41:18Z" name: switch-graph-e938d-798f566f4f-4lmhz.18a96c2f467a4079 namespace: kserve-ci-e2e-test resourceVersion: "15339" uid: a38516db-8228-4268-9593-12f856f65a77 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:49:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e938d} kind: Pod name: switch-graph-e938d-798f566f4f-4lmhz namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: d92fa55c-f6a7-4353-b1d9-a8935574e0fd kind: Event lastTimestamp: "2026-04-24T22:49:31Z" message: Stopping container switch-graph-e938d metadata: creationTimestamp: "2026-04-24T22:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:49:31Z" name: switch-graph-e938d-798f566f4f-4lmhz.18a96ca23094e38a namespace: kserve-ci-e2e-test resourceVersion: "18822" uid: bf012ce3-ae3d-48ec-ba6d-beb5922a4fbe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-24T22:49:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e938d} kind: Pod name: switch-graph-e938d-798f566f4f-4lmhz namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: d92fa55c-f6a7-4353-b1d9-a8935574e0fd kind: Event lastTimestamp: "2026-04-24T22:50:00Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-24T22:49:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-24T22:50:00Z" name: switch-graph-e938d-798f566f4f-4lmhz.18a96ca2ede8f908 namespace: kserve-ci-e2e-test resourceVersion: "19157" uid: 35638077-e01b-4277-82a7-d13ee1779f63 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-161.ec2.internal source: component: kubelet host: ip-10-0-133-161.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-e938d-798f566f4f namespace: kserve-ci-e2e-test resourceVersion: "15304" uid: f4374a22-1051-44a3-b4f5-fa532285d73b kind: Event lastTimestamp: "2026-04-24T22:41:17Z" message: 'Created pod: switch-graph-e938d-798f566f4f-4lmhz' metadata: creationTimestamp: "2026-04-24T22:41:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:17Z" name: switch-graph-e938d-798f566f4f.18a96c2f04c8b63a namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: a90a47ef-2063-4157-956c-e0caf320e022 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-e938d namespace: kserve-ci-e2e-test resourceVersion: "15303" uid: 9e32ef62-31f9-4870-9bb4-862e2091c02a kind: Event lastTimestamp: "2026-04-24T22:41:17Z" message: Scaled up replica set switch-graph-e938d-798f566f4f from 0 to 1 metadata: creationTimestamp: "2026-04-24T22:41:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:17Z" name: switch-graph-e938d.18a96c2f03a29a83 namespace: kserve-ci-e2e-test resourceVersion: "15306" uid: 2e517963-8092-4776-be8b-7048ceca58d0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-e938d namespace: kserve-ci-e2e-test resourceVersion: "15319" uid: 82457223-d75d-4ce4-b2fc-422bfa0f880b kind: Event lastTimestamp: "2026-04-24T22:41:17Z" message: InferenceGraph [switch-graph-e938d] is Ready metadata: creationTimestamp: "2026-04-24T22:41:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-24T22:41:17Z" name: switch-graph-e938d.18a96c2f07e2e4af namespace: kserve-ci-e2e-test resourceVersion: "15321" uid: 10509d28-1d02-4904-bf0e-222a6c5d0bb4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-e938d namespace: kserve-ci-e2e-test resourceVersion: "15305" uid: 0ac05512-6adc-403f-8931-8dd308d12c09 kind: Event lastTimestamp: "2026-04-24T22:41:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:41:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:32Z" name: switch-graph-e938d.18a96c3282c0b87b namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: 1e5cd323-0e9b-4698-b164-a75b2a1883c8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-e938d namespace: kserve-ci-e2e-test resourceVersion: "15305" uid: 0ac05512-6adc-403f-8931-8dd308d12c09 kind: Event lastTimestamp: "2026-04-24T22:41:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-24T22:41:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:32Z" name: switch-graph-e938d.18a96c32830ac462 namespace: kserve-ci-e2e-test resourceVersion: "15459" uid: 20ed7bd1-9d66-4b71-a4e7-23b5dd0a0e74 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-e938d namespace: kserve-ci-e2e-test resourceVersion: "15457" uid: 0ac05512-6adc-403f-8931-8dd308d12c09 kind: Event lastTimestamp: "2026-04-24T22:41:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:41:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:47Z" name: switch-graph-e938d.18a96c3601be3bd8 namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: 02b52d2c-c47c-41b9-8882-ca924c4fd36c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-24T22:41:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-e938d namespace: kserve-ci-e2e-test resourceVersion: "15457" uid: 0ac05512-6adc-403f-8931-8dd308d12c09 kind: Event lastTimestamp: "2026-04-24T22:41:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-24T22:41:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-24T22:41:47Z" name: switch-graph-e938d.18a96c36020cb29a namespace: kserve-ci-e2e-test resourceVersion: "15590" uid: 775197cc-5021-434f-8b79-be318a4e5ddd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "32807"